Attachment #65941 for bug #185580





public class Utils {

	public final static double STANDARD_ERROR_THRESHOLD = 0.03; // 3%
	public final static double STANDARD_ERROR_THRESHOLD = 0.02; // 2%
	static final NumberFormat PERCENT_FORMAT = NumberFormat.getPercentInstance();
	static {
		PERCENT_FORMAT.setMaximumFractionDigits(1);
	}
	public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);
	public final static String TTEST_MODERATION_MESSAGE="Student's t-test failed at the 90% level but it's moderated with standard error lower than "+STANDARD_ERROR_THRESHOLD_STRING;
	public final static String STANDARD_ERROR_MESSAGE="Standard error on this test is higher than "+STANDARD_ERROR_THRESHOLD_STRING;
	public final static String OK_IMAGE="OK.gif";
	public final static String OK_IMAGE_ERR="OK_err.gif";
	public final static String OK_IMAGE_TTEST="OK_ttest.gif";
	public final static String OK_IMAGE_WARN="OK_caution.gif";
	public final static String FAIL_IMAGE="FAIL.gif";
	public final static String FAIL_IMAGE_TTEST="FAIL_ttest.gif";
	public final static String FAIL_IMAGE_ERR="FAIL_err.gif";
	public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";
	public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";
	public final static int OK = 0;
	public final static int ERR = 0x1;
	public final static int TTEST = 0x2;
	public final static int DEV = 0x4;
	private final static int NOT_SIGNIFICANT = ERR | TTEST;

	/**
	 * @param dimension

	}

	/**
	 * Return &lt;html&gt;&lt;head&gt;&lt;meta http-equiv="Content-Type"
	 *         content="text/html; charset=iso-8859-1"&gt;
	 */
	public static String HTML_OPEN = "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\">";

	/**
	 * Return "&lt;/html&gt;".
	 */
	public static String HTML_CLOSE = "</html>";


		String description;

		/**
		 *
		 * @param name
		 *            the value specifed for the key config in the
		 *            eclipse.perf.config system.property key value listings.

			this.name = name;
			this.description = description;
		}

		public String getName() {
			return name;
		}

		public boolean descriptionMatches(String descr) {
			return descr.equals(this.description);
		}
	}


	/**
	 * Queries database with variation composed of buildIdPattern, config and
	 * jvm.
	 *
	 * @return Array of scenarios.
	 */
	public static Scenario[] getScenarios(String buildIdPattern, String scenarioPattern, String config, String jvm) {


	/**
	 * Creates a Variations object using build id pattern, config and jvm.
	 *
	 * @param buildIdPattern
	 * @param config
	 * @param jvm


	/**
	 * Utility method to copy a file.
	 *
	 * @param src the source file.
	 * @param dest the destination.
	 * @param dest -
	 *            the destination.
	 */
	private static void copyFile(File src, File dest) {

		try {
			InputStream in = new FileInputStream(src);

			e.printStackTrace();
		}
	}
	public static void copyImages(File images, File output) {
		copyFile(new File(images, Utils.FAIL_IMAGE), new File(output, Utils.FAIL_IMAGE));
		copyFile(new File(images, Utils.FAIL_IMAGE_EXPLAINED), new File(output, Utils.FAIL_IMAGE_EXPLAINED));
		copyFile(new File(images, Utils.FAIL_IMAGE_WARN), new File(output, Utils.FAIL_IMAGE_WARN));
		copyFile(new File(images, Utils.OK_IMAGE), new File(output, Utils.OK_IMAGE));
		copyFile(new File(images, Utils.OK_IMAGE_WARN), new File(output, Utils.OK_IMAGE_WARN));
	}
	public static void copyScripts(File scripts, File output) {
		copyFile(new File(scripts, "ToolTip.css"), new File(output, "ToolTip.css"));
		copyFile(new File(scripts, "ToolTip.js"), new File(output, "ToolTip.js"));
	}

	/**
	 * Returns a LineGraph object representing measurements for a scenario over
	 * builds.
	 *
	 * @param t -
	 *            the scenario object.
	 * @param dimensionName -

						Color color = black;
						if (buildID.startsWith("N"))
							color = yellow;

						graph.addItem("main", label, dim.getDisplayValue(value), value, color, true, getDateFromBuildID(buildID), true);
						continue;
					}

						continue;
					} else if (buildID.startsWith("N"))
							continue;

					for (int i=0;i<currentBuildIdPrefixes.size();i++){
						if (buildID.startsWith(currentBuildIdPrefixes.get(i).toString())&&!currentFound) {
							graph.addItem("main", buildID, dim.getDisplayValue(value), value, black, false, getDateFromBuildID(buildID), false);


	/**
	 * Prints a LineGraph object as a gif
	 *
	 * @param p -
	 *            the LineGraph object.
	 * @param output -

	/**
	 * Utility method which returns HTML code representing an image map for a
	 * LineGraph object.
	 *
	 * @param p -
	 *            the LineGraph object.
	 * @param imageSource -


	/**
	 * A utility which returns the index of a given buildId in an array.
	 *
	 * @param timeSeriesLabels -
	 *            array of buildIds
	 * @param buildId -


	/**
	 * Returns the date/time from the build id in format yyyymmddhm
	 *
	 * @param buildId
	 * @return date/time in format YYYYMMDDHHMM, ie. 200504060010
	 */

			out = new PrintWriter(new FileWriter(new File(outputFile)));
			out.println(Utils.HTML_OPEN + "</head><body>\n");
			out.println("<h3>Summary of Elapsed Process Variation Coefficients</h3>\n"+
		"<p> This table provides a bird's eye view of variability in elapsed process times\n"+
		  "for baseline and current build stream performance scenarios." +
		  " This summary is provided to facilitate the identification of scenarios that should be examined due to high variability." +
		  "The variability for each scenario is expressed as a <a href=\"http://en.wikipedia.org/wiki/Coefficient_of_variation\">coefficient\n"+
		  "of variation</a> (CV). The CV is calculated by dividing the <b>standard deviation\n"+
		  "of the elapse process time over builds</b> by the <b>average elapsed process\n"+
		  "time over builds</b> and multiplying by 100.\n"+
		"</p><p>High CV values may be indicative of any of the following:<br></p>\n"+
		"<ol><li> an unstable performance test. </li>\n"+
		  "<ul><li>may be evidenced by an erratic elapsed process line graph.<br><br></li></ul>\n"+

		  "<li>unstable testing hardware.\n" +
		  "<ul><li>consistent higher CV values for one test configuration as compared to others across" +
		  " scenarios may be related to hardward problems.</li></ul></li></ol>\n"+
		"<p> Scenarios are listed in alphabetical order in the far right column. A scenario's\n"+
		  "variation coefficients (CVs) are in columns to the left for baseline and current\n"+
		  "build streams for each test configuration. Scenarios with CVs > 10% are highlighted\n"+
		  "in yellow (10%<CV>&lt;CV<20%) and orange(CV>20%). </p>\n"+
		"<p> Each CV value links to the scenario's detailed results to allow viewers to\n"+
		  "investigate the variability.</p>\n");

			Hashtable cvTable = (Hashtable) rawDataTables.get(scenarios[0]);
			String[] configNames = (String[]) cvTable.keySet().toArray(new String[cvTable.size()]);
			Arrays.sort(configNames);


		  	int configColumns=configNames.length/2;
			out.println("<table border=\"1\"><tr>" +
					    "<td colspan=\""+configColumns+"\"><b>Baseline CVs</b></td>"+
					    "<td colspan=\""+configColumns+"\"><b>Current Build Stream CVs</b></td>"+
					    "<td rowspan=\"2\"><b>Scenario Name</b></td>"+
					    "</tr><tr>");


			for (int i = 0; i < configNames.length; i++) {
				//configNames here have prefix cConfig- or bConfig- depending on whether the data comes from
				//current build stream data or baseline data.
				out.print("<td>" + ((ConfigDescriptor)configDescriptors.get(configNames[i].substring(8))).description + "</td>");
			}

				Hashtable aCvTable = (Hashtable) rawDataTables.get(scenarios[i]);
				String scenario = scenarios[i];
				String scenarioFile=scenario.replace('#', '.').replace(':', '_').replace('\\', '_')+".html";

				for (int j = 0; j < configNames.length; j++) {
					String url=configNames[j].substring(8)+"/"+scenarioFile;
					if (aCvTable.get(configNames[j]) == null) {


	public static boolean hasConfidentResult(Variations variations, String scenarioName, String baseline, String config) {
	    double[] resultStats = resultStats(variations, scenarioName, baseline, config);
	    return (confidenceLevel(resultStats) & ERR) == 0;
	    	return resultStats[1] < 0 || resultStats[0] < resultStats[1] || resultStats[2] < STANDARD_ERROR_THRESHOLD;
	    }
	    return true;
    }
	public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) {
		return failureMessage(resultStats(variations, scenarioName, baseline, config), true);
	}
	public static String failureMessage(double[] resultStats, boolean full) {
		StringBuffer buffer = new StringBuffer();
		if (full & (confidenceLevel(resultStats) & ERR) != 0) {
			buffer.append("*** WARNING ***  ");
 			buffer.append(STANDARD_ERROR_MESSAGE);
		}
		if (!full) buffer.append("<small><font color=\"gray\">  ");
		if (resultStats == null) {
			buffer.append('?');
		} else {
			double deviation = resultStats[3]==0 ? 0 : -resultStats[3];
			if (deviation > 0) {
				buffer.append('+');
			}
 			buffer.append(PERCENT_FORMAT.format(deviation));
 			buffer.append(" (&#177;");
 			buffer.append(PERCENT_FORMAT.format(resultStats[2]));
 			buffer.append(")");
				break;
			case TTEST|ERR:
				buffer = new StringBuffer();
				if (prefix) buffer.append("*** WARNING ***  ");
     			buffer.append(TTEST_FAILURE_MESSAGE);
				break;
			default:
				return null;
		}
		if (!full) buffer.append("</font></small>");
		return buffer.toString();
	}
	public static int confidenceLevel(double[] resultStats) {

	public static String getImage(int confidence, boolean hasExplanation) {
	    boolean scenarioFailed = (confidence & DEV) != 0;
	    String image = null;

	    if (scenarioFailed) {
	    	if (hasExplanation) {
		    	image = FAIL_IMAGE_EXPLAINED;
		    } else if ((confidence & ERR) != 0) {
    			image = FAIL_IMAGE_WARN;
		    } else {
    			image = FAIL_IMAGE;
		    		default:
		    			image = FAIL_IMAGE;
		    			break;
		    		case TTEST:
		    			image = FAIL_IMAGE_TTEST;
		    			break;
		    		case ERR:
		    			image = FAIL_IMAGE_ERR;
		    			break;
		    		case NOT_SIGNIFICANT:
		    			image = FAIL_IMAGE_WARN;
		    			break;
		    	}
		    }
	    } else if ((confidence & ERR) != 0) {
   			image = OK_IMAGE_WARN;
	    } else {
   			image = OK_IMAGE;
	    			break;
	    		case TTEST:
	    			image = OK_IMAGE_TTEST;
	    			break;
	    		case ERR:
	    			image = OK_IMAGE_ERR;
	    			break;
	    		case NOT_SIGNIFICANT:
	    			image = OK_IMAGE_WARN;
	    			break;
	    	}
	    }
	    return image;
    }




						configNames.add(configs[j]);

					double[] resultStats = Utils.resultStats(variations, scenarioName, baseline, configs[j]);
					String failureMessage = Utils.failureMessage(resultStats, false);
					int confidenceLevel = Utils.confidenceLevel(resultStats);
					
					boolean hasScenarioFailure = failureMessages[j] != null && failureMessages[j].indexOf(configs[j]) != -1; // ensure correct failure message relates to config
					StringBuffer buffer = new StringBuffer();
					if (hasScenarioFailure) {
						buffer.append(failureMessages[j]);
						if (scenarioStatus.hasSlowDownExplanation) {
							buffer.append(" - Explanation comment: ");
							buffer.append(scenarioComment);
								buf.append(scenarioComment);
							}
							confidenceLevel |= Utils.DEV;
						}
						confidenceLevel |= Utils.DEV;
							if (buf.length() > 0) buf.append("\n - ");
							buf.append(failureMessage);
						}
						failureMessage = buf.toString();
					} else {
						failureMessage = "";
					}

					scenarioStatus.configStatus.put(configs[j], new Integer(confidenceLevel));
					scenarioStatus.statusMap.put(configs[j], new Object[] { buffer.toString(), resultStats });
				}
			}
			
			String label=null;
			htmlTable=htmlTable.concat("<br><h4>Scenario Status</h4>\n" +
				"Click on test box corresponding image (<img src=\"FAIL.gif\"> or <img src=\"OK.gif\">) for detailed results.<br>\n" +
				"For all other images than  <img src=\"OK.gif\">, fly over it to get corresponding error/warning/info message.<br>\n" +
				"The images color legend is:\n" +
				"<ul>\n" +
				"<li>Green (<img src=\""+Utils.OK_IMAGE+"\">): mark a <b>successful result</b>, which means this test has neither significant performance regression nor significant standard error</li>\n" +
				"<li>Red (<img src=\""+Utils.FAIL_IMAGE+"\">): mark a <b>failing result</b>, which means this test shows a significant performance regression (more than 10%)</li>\n" +
				"<li>Grayed (<img src=\""+Utils.FAIL_IMAGE_EXPLAINED+"\">): mark a <b>failing result</b> (see above) with a comment explaining this degradation.</li>\n" +
				"<li>Yellow (<img src=\""+Utils.FAIL_IMAGE_WARN+"\"> or <img src=\""+Utils.OK_IMAGE_WARN+"\">): mark a <b>failing or successful result</b> with a significant standard error (more than "+Utils.STANDARD_ERROR_THRESHOLD_STRING+")</li>\n" +
				"<li>\"n/a\": mark a test for with <b>no</b> performance results</li>\n" +
				"</ul>\n" +
				"For failing tests, value of deviation with its standard error is added at the beginning of the error message you can see flying over the corresponding image.<br>\n" +
				"Follow the link on test box corresponding image for detailed results.<br><br>\n");

			htmlTable=htmlTable.concat("<table border=\"1\"><tr><td><h4>All "+scenarios.length+" scenarios</h4></td>\n");
			for (int i= 0; i < configNames.size(); i++){
				label=configNames.get(i).toString();

					String message=null;
					String configName=configNames.get(i).toString();
					String aUrl=configName;
					double[] resultStats = null;
					if(status.statusMap.get(configName)!=null){
						Object[] statusInfo = (Object[]) status.statusMap.get(configName);
						message = (String) statusInfo[0];
						resultStats = (double[]) statusInfo[1];
					}

					if (status.statusMap.containsKey(configName)){
						int confidence = ((Integer) status.configStatus.get(configName)).intValue();
						String image = Utils.getImage(confidence, status.hasSlowDownExplanation);
						StringBuffer html = new StringBuffer("\n<td><a ");
						if ((confidence & Utils.DEV) == 0 || message.length() == 0){
							// write deviation with error in table when test pass
							html.append("href=\"");
							html.append(aUrl);
							html.append('/');
							html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
							html.append(".html\">\n<img border=\"0\" src=\"");
							html.append(image);
							html.append("\"/></a>");
						} else {
							// create message with tooltip text including deviation with error plus failure message
							jsIdCount+=1;
							html.append("class=\"tooltipSource\" onMouseover=\"show_element('toolTip");
							html.append(jsIdCount);
							html.append("')\" onMouseout=\"hide_element('toolTip");
							html.append(jsIdCount);
							html.append("')\" \nhref=\"");
							html.append(aUrl);
							html.append('/');
							html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
							html.append(".html\">\n<img border=\"0\" src=\"");
							html.append(image);
							html.append("\"/>\n<span class=\"hidden_tooltip\" id=\"toolTip");
							html.append(jsIdCount);
							html.append("\">");
							html.append(message);
							html.append("</span></a>");
						}
						html.append(Utils.failureMessage(resultStats, false));
						html.append("</td>");
						htmlTable=htmlTable.concat(html.toString());
					}else{
						htmlTable=htmlTable.concat("<td>n/a</td>");
					}

Lines 131-137 Link Here

(-)src/org/eclipse/test/performance/ui/ScenarioResults.java (-1 / +1 lines)
131	ps.println("<title>" + t.getScenarioName() + "(" + configDescriptor.description + ")" + "</title></head>"); //$NON-NLS-1$	131	ps.println("<title>" + t.getScenarioName() + "(" + configDescriptor.description + ")" + "</title></head>"); //$NON-NLS-1$
132	ps.println("<h4>Scenario: " + t.getScenarioName() + " (" + configDescriptor.description + ")</h4><br>"); //$NON-NLS-1$ //$NON-NLS-2$	132	ps.println("<h4>Scenario: " + t.getScenarioName() + " (" + configDescriptor.description + ")</h4><br>"); //$NON-NLS-1$ //$NON-NLS-2$
133		133
134	String failureMessage = Utils.failureMessage(variations, t.getScenarioName(),baseline,configDescriptor.name, true/prefix/);	134	String failureMessage = Utils.failureMessage(variations, t.getScenarioName(),baseline,configDescriptor.name);
135	if (failureMessage != null){	135	if (failureMessage != null){
136	ps.println("<table><tr><td><b>"+failureMessage+"</td></tr></table>\n");	136	ps.println("<table><tr><td><b>"+failureMessage+"</td></tr></table>\n");
137	}	137	}




package org.eclipse.test.performance.ui;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.net.URL;


		if (images!=null) {
			images= Platform.resolve(images);
			Utils.copyImages(new File(images.getPath()), new File(output));
			Utils.copyFile(new File(images.getPath(), "FAIL_ttest.gif"), output + "/FAIL_ttest.gif");
			Utils.copyFile(new File(images.getPath(), "FAIL_err.gif"), output + "/FAIL_err.gif");
			Utils.copyFile(new File(images.getPath(), "FAIL_greyed.gif"), output+"/FAIL_greyed.gif");
			Utils.copyFile(new File(images.getPath(), "FAIL_caution.gif"), output+"/FAIL_caution.gif");
			Utils.copyFile(new File(images.getPath(), "OK.gif"),output + "/OK.gif");
			Utils.copyFile(new File(images.getPath(), "OK_err.gif"),output + "/OK_err.gif");
			Utils.copyFile(new File(images.getPath(), "OK_ttest.gif"),output + "/OK_ttest.gif");
			Utils.copyFile(new File(images.getPath(), "OK_caution.gif"),output + "/OK_caution.gif");
		}
		if (scripts!=null){
			scripts= Platform.resolve(scripts);
			Utils.copyScripts(new File(scripts.getPath()), new File(output));
			Utils.copyFile(new File(scripts.getPath(), "ToolTip.js"),output+"/ToolTip.js");
		}
		
		// print fingerprint/scenario status pages




     * XXX the current implementation assumes that the standard deviations are sufficiently similar.
     * </p>
     * 
     * @param refSeries the time series containing the first data set
     * @param index1 the index into <code>series1</code> for the first data set
     * @param testSeries the time series containing the second data set
     * @param index2 the index into <code>series2</code> for the second data set
     * @param percentile the percentile level to use
     * @return <code>true</code> if the null hypothesis is rejected on the <code>percentile</code>
     *         level, <code>false</code> if it cannot be rejected based on the given data
     */
    public static double[] statisticsForTimeSeries(TimeSeries refSeries, int index1, TimeSeries testSeries, int index2, Percentile percentile) {
        // see http://bmj.bmjjournals.com/collections/statsbk/7.shtml
    
    	double[] values = new double[] { refSeries.getValue(index1), testSeries.getValue(index2) };
    	long[] counts = new long[] { refSeries.getCount(index1), testSeries.getCount(index2) };
    	double[] stddevs = new double[] { refSeries.getStddev(index1), testSeries.getStddev(index2) };
		double t = studentTtest(values, stddevs, counts, percentile);
		return t == -1 || getStudentsT((int) (counts[0] + counts[1] - 2), percentile) < t;
    }

    public static double[] statisticsForTimeSeries(TimeSeries series1, int index1, TimeSeries series2, int index2, Percentile percentile) {
        // see http://bmj.bmjjournals.com/collections/statsbk/7.shtml
    
    	double[] values = new double[] { series1.getValue(index1), series2.getValue(index2) };
    	long[] counts = new long[] { series1.getCount(index1), series2.getCount(index2) };
    	double[] stddevs = new double[] { series1.getStddev(index1), series2.getStddev(index2) };
		double ttest = studentTtest(values, stddevs, counts, percentile);
		return new double[] {
			getStudentsT((int) (counts[0] + counts[1] - 2), percentile),
			ttest,
			standardError(values, stddevs, counts),
			deviation(values),
		};
    }


        return -1;
    }

    public static double deviation(double[] values) {
    	return (values[1] - values[0]) / values[0];
    }

    public static double standardError(double[] values, double[] stddevs, long[] counts) {
    	return Math.sqrt((stddevs[0] * stddevs[0] / counts[0]) + (stddevs[1] * stddevs[1] / counts[1])) / values[0];
    }

    /**
     * The (two-tailed) T-table. [degrees_of_freedom][percentile]

Lines 399-408 Link Here

(-)src/org/eclipse/test/internal/performance/db/DB.java (-13 / +58 lines)
399	if (fSQL == null)	399	if (fSQL == null)
400	return null;	400	return null;
401		401
		402	long start = System.currentTimeMillis();
		403	if (DEBUG) System.out.print(" - query data points from DB for scenario "+scenarioName+"..."); //$NON-NLS-1$ //$NON-NLS-2$
402	ResultSet rs= null;	404	ResultSet rs= null;
403	try {	405	try {
404	ArrayList dataPoints= new ArrayList();	406	ArrayList dataPoints= new ArrayList();
405	rs= fSQL.queryDataPoints(variations, scenarioName);	407	rs= fSQL.queryDataPoints(variations, scenarioName);
		408	if (DEBUG) {
		409	long time = System.currentTimeMillis();
		410	System.out.println("done in "+(time-start)+"ms"); //$NON-NLS-1$ //$NON-NLS-2$
		411	start = time;
		412	}
406	while (rs.next()) {	413	while (rs.next()) {
407	int datapoint_id= rs.getInt(1);	414	int datapoint_id= rs.getInt(1);
408	int step= rs.getInt(2);	415	int step= rs.getInt(2);
Lines 426-432 Link Here
426	rs.close();	433	rs.close();
427		434
428	int n= dataPoints.size();	435	int n= dataPoints.size();
429	if (DEBUG) System.out.println("query resulted in " + n + " datapoints from DB"); //$NON-NLS-1$ //$NON-NLS-2$	436	if (DEBUG) {
		437	long time = System.currentTimeMillis();
		438	System.out.println(" + " + n + " datapoints created in "+(time-start)+"ms"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
		439	}
430	return (DataPoint[])dataPoints.toArray(new DataPoint[n]);	440	return (DataPoint[])dataPoints.toArray(new DataPoint[n]);
431		441
432	} catch (SQLException e) {	442	} catch (SQLException e) {
Lines 449-454 Link Here
449	private String[] internalQueryScenarioNames(Variations variations, String scenarioPattern) {	459	private String[] internalQueryScenarioNames(Variations variations, String scenarioPattern) {
450	if (fSQL == null)	460	if (fSQL == null)
451	return null;	461	return null;
		462	long start = System.currentTimeMillis();
		463	if (DEBUG) System.out.print(" - query scenario names from DB for scenario pattern '"+scenarioPattern+"'..."); //$NON-NLS-1$ //$NON-NLS-2$
452	ResultSet result= null;	464	ResultSet result= null;
453	try {	465	try {
454	result= fSQL.queryScenarios(variations, scenarioPattern);	466	result= fSQL.queryScenarios(variations, scenarioPattern);
Lines 461-472 Link Here
461	PerformanceTestPlugin.log(e);	473	PerformanceTestPlugin.log(e);
462		474
463	} finally {	475	} finally {
464	if (result != null)	476	if (result != null) {
465	try {	477	try {
466	result.close();	478	result.close();
467	} catch (SQLException e1) {	479	} catch (SQLException e1) {
468	// ignored	480	// ignored
469	}	481	}
		482	}
		483	if (DEBUG) {
		484	long time = System.currentTimeMillis();
		485	System.out.println("done in "+(time-start)+"ms"); //$NON-NLS-1$ //$NON-NLS-2$
		486	}
470	}	487	}
471	return null;	488	return null;
472	}	489	}
Lines 477-482 Link Here
477	private void internalQueryDistinctValues(List values, String seriesKey, Variations variations, String scenarioPattern) {	494	private void internalQueryDistinctValues(List values, String seriesKey, Variations variations, String scenarioPattern) {
478	if (fSQL == null)	495	if (fSQL == null)
479	return;	496	return;
		497	long start = System.currentTimeMillis();
		498	if (DEBUG) System.out.print(" - query distinct values from DB for scenario pattern '"+scenarioPattern+"'..."); //$NON-NLS-1$ //$NON-NLS-2$
480	ResultSet result= null;	499	ResultSet result= null;
481	try {	500	try {
482	result= fSQL.queryVariations(variations.toExactMatchString(), scenarioPattern);	501	result= fSQL.queryVariations(variations.toExactMatchString(), scenarioPattern);
Lines 491-521 Link Here
491	PerformanceTestPlugin.log(e);	510	PerformanceTestPlugin.log(e);
492		511
493	} finally {	512	} finally {
494	if (result != null)	513	if (result != null) {
495	try {	514	try {
496	result.close();	515	result.close();
497	} catch (SQLException e1) {	516	} catch (SQLException e1) {
498	// ignored	517	// ignored
499	}	518	}
		519	}
		520	if (DEBUG) {
		521	long time = System.currentTimeMillis();
		522	System.out.println("done in "+(time-start)+"ms"); //$NON-NLS-1$ //$NON-NLS-2$
		523	}
500	}	524	}
501	}	525	}
502		526
503	private SummaryEntry[] internalQuerySummaries(Variations variationPatterns, String scenarioPattern) {	527	private SummaryEntry[] internalQuerySummaries(Variations variationPatterns, String scenarioPattern) {
504	if (fSQL == null)	528	if (fSQL == null)
505	return null;	529	return null;
		530	long start = System.currentTimeMillis();
		531	if (DEBUG) System.out.print(" - query summaries from DB for scenario pattern '"+scenarioPattern+"'..."); //$NON-NLS-1$ //$NON-NLS-2$
		532	ResultSet result = null;
506	try {	533	try {
507	List fingerprints= new ArrayList();	534	List fingerprints= new ArrayList();
508	ResultSet rs;
509	if (scenarioPattern != null)	535	if (scenarioPattern != null)
510	rs= fSQL.querySummaryEntries(variationPatterns, scenarioPattern);	536	result= fSQL.querySummaryEntries(variationPatterns, scenarioPattern);
511	else	537	else
512	rs= fSQL.queryGlobalSummaryEntries(variationPatterns);	538	result= fSQL.queryGlobalSummaryEntries(variationPatterns);
513	while (rs.next()) {	539	while (result.next()) {
514	String scenarioName= rs.getString(1);	540	String scenarioName= result.getString(1);
515	String shortName= rs.getString(2);	541	String shortName= result.getString(2);
516	int dim_id= rs.getInt(3);	542	int dim_id= result.getInt(3);
517	boolean isGlobal= rs.getShort(4) == 1;	543	boolean isGlobal= result.getShort(4) == 1;
518	int comment_id= rs.getInt(5);	544	int comment_id= result.getInt(5);
519	int commentKind= 0;	545	int commentKind= 0;
520	String comment= null;	546	String comment= null;
521	if (comment_id != 0) {	547	if (comment_id != 0) {
Lines 530-535 Link Here
530	return (SummaryEntry[])fingerprints.toArray(new SummaryEntry[fingerprints.size()]);	556	return (SummaryEntry[])fingerprints.toArray(new SummaryEntry[fingerprints.size()]);
531	} catch (SQLException e) {	557	} catch (SQLException e) {
532	PerformanceTestPlugin.log(e);	558	PerformanceTestPlugin.log(e);
		559	} finally {
		560	if (result != null) {
		561	try {
		562	result.close();
		563	} catch (SQLException e1) {
		564	// ignored
		565	}
		566	}
		567	if (DEBUG) {
		568	long time = System.currentTimeMillis();
		569	System.out.println("done in "+(time-start)+"ms"); //$NON-NLS-1$ //$NON-NLS-2$
		570	}
533	}	571	}
534	return null;	572	return null;
535	}	573	}
Lines 598-603 Link Here
598	private Map internalQueryFailure(String scenarioPattern, Variations variations) {	636	private Map internalQueryFailure(String scenarioPattern, Variations variations) {
599	if (fSQL == null)	637	if (fSQL == null)
600	return null;	638	return null;
		639	long start = System.currentTimeMillis();
		640	if (DEBUG) System.out.print(" - query failure from DB for scenario pattern '"+scenarioPattern+"'..."); //$NON-NLS-1$ //$NON-NLS-2$
601	ResultSet result= null;	641	ResultSet result= null;
602	try {	642	try {
603	Map map= new HashMap();	643	Map map= new HashMap();
Lines 612-623 Link Here
612	PerformanceTestPlugin.log(e);	652	PerformanceTestPlugin.log(e);
613		653
614	} finally {	654	} finally {
615	if (result != null)	655	if (result != null) {
616	try {	656	try {
617	result.close();	657	result.close();
618	} catch (SQLException e1) {	658	} catch (SQLException e1) {
619	// ignored	659	// ignored
620	}	660	}
		661	}
		662	if (DEBUG) {
		663	long time = System.currentTimeMillis();
		664	System.out.println("done in "+(time-start)+"ms"); //$NON-NLS-1$ //$NON-NLS-2$
		665	}
621	}	666	}
622	return null;	667	return null;
623	}	668	}

Return to bug 185580

Lines 87-134 Link Here

(-)src/org/eclipse/test/performance/ui/ScenarioStatusTable.java (-48 / +53 lines)
87	configNames.add(configs[j]);	87	configNames.add(configs[j]);
88		88
89	double[] resultStats = Utils.resultStats(variations, scenarioName, baseline, configs[j]);	89	double[] resultStats = Utils.resultStats(variations, scenarioName, baseline, configs[j]);
90	String failureMessage = Utils.failureMessage(resultStats, false);
91	int confidenceLevel = Utils.confidenceLevel(resultStats);	90	int confidenceLevel = Utils.confidenceLevel(resultStats);
92		91
93	boolean hasScenarioFailure = failureMessages[j] != null && failureMessages[j].indexOf(configs[j]) != -1; // ensure correct failure message relates to config	92	boolean hasScenarioFailure = failureMessages[j] != null && failureMessages[j].indexOf(configs[j]) != -1; // ensure correct failure message relates to config
94	if (hasScenarioFailure \|\| failureMessage != null){	93	StringBuffer buffer = new StringBuffer();
95	StringBuffer buf= new StringBuffer();	94	if (hasScenarioFailure) {
96	if (hasScenarioFailure) {	95	buffer.append(failureMessages[j]);
97	buf.append(failureMessages[j]);	96	if (scenarioStatus.hasSlowDownExplanation) {
98	if (scenarioStatus.hasSlowDownExplanation) {	97	buffer.append(" - Explanation comment: ");
99	buf.append("\n - Explanation comment: ");	98	buffer.append(scenarioComment);
100	buf.append(scenarioComment);
101	}
102	confidenceLevel \|= Utils.DEV;
103	}	99	}
104	if (failureMessage != null) {	100	confidenceLevel \|= Utils.DEV;
105	if (buf.length() > 0) buf.append("\n - ");
106	buf.append(failureMessage);
107	}
108	failureMessage = buf.toString();
109	} else {
110	failureMessage = "";
111	}	101	}
112		102
113	scenarioStatus.configStatus.put(configs[j], new Integer(confidenceLevel));	103	scenarioStatus.configStatus.put(configs[j], new Integer(confidenceLevel));
114	scenarioStatus.statusMap.put(configs[j], failureMessage);	104	scenarioStatus.statusMap.put(configs[j], new Object[] { buffer.toString(), resultStats });
115	}	105	}
116	}	106	}
117		107
118	String label=null;	108	String label=null;
119	htmlTable=htmlTable.concat("<br><h4>Scenario Status</h4>\n" +	109	htmlTable=htmlTable.concat("<br><h4>Scenario Status</h4>\n" +
120	"Click on test box corresponding image (<img src=\"FAIL.gif\"> or <img src=\"OK.gif\">) for detailed results.<br>\n" +
121	"For all other images than <img src=\"OK.gif\">, fly over it to get corresponding error/warning/info message.<br>\n" +
122	"The images color legend is:\n" +	110	"The images color legend is:\n" +
123	"<ul>\n" +	111	"<ul>\n" +
124	"<li>Red (<img src=\"FAIL.gif\">): indicates that the assert condition failed for the test</li>\n" +	112	"<li>Green (<img src=\""+Utils.OK_IMAGE+"\">): mark a <b>successful result</b>, which means this test has neither significant performance regression nor significant standard error</li>\n" +
125	"<li>Grayed (<img src=\"FAIL_greyed.gif\">): mark explainable degradations.</li>\n" +	113	"<li>Red (<img src=\""+Utils.FAIL_IMAGE+"\">): mark a <b>failing result</b>, which means this test shows a significant performance regression (more than 10%)</li>\n" +
126	"<li>Light (<img src=\"FAIL_err.gif\"> or <img src=\"OK_err.gif\">): mark results with standard error higher than "+Utils.STANDARD_ERROR_THRESHOLD_STRING+"</li>\n" +	114	"<li>Grayed (<img src=\""+Utils.FAIL_IMAGE_EXPLAINED+"\">): mark a <b>failing result</b> (see above) with a comment explaining this degradation.</li>\n" +
127	"<li>Blue (<img src=\"FAIL_ttest.gif\"> or <img src=\"OK_ttest.gif\">): mark results where Student's t-test failed (see below) but was moderated by the fact that the error is less than "+Utils.STANDARD_ERROR_THRESHOLD_STRING+"</li>\n" +	115	"<li>Yellow (<img src=\""+Utils.FAIL_IMAGE_WARN+"\"> or <img src=\""+Utils.OK_IMAGE_WARN+"\">): mark a <b>failing or successful result</b> with a significant standard error (more than "+Utils.STANDARD_ERROR_THRESHOLD_STRING+")</li>\n" +
128	"<li>Yellow (<img src=\"FAIL_caution.gif\"> or <img src=\"OK_caution.gif\">): mark results where Student's t-test failed which means that there is not enough evidence to reject the null hypothesis at the 90% level.<br>\n" +	116	"<li>\"n/a\": mark a test for with <b>no</b> performance results</li>\n" +
129	"<li>\"n/a\": mark not available results.</li>\n" +	117	"</ul>\n" +
130	"</ul><br>\n");	118	"For failing tests, value of deviation with its standard error is added at the beginning of the error message you can see flying over the corresponding image.<br>\n" +
131		119	"Follow the link on test box corresponding image for detailed results.<br><br>\n");
		120
132	htmlTable=htmlTable.concat("<table border=\"1\"><tr><td><h4>All "+scenarios.length+" scenarios</h4></td>\n");	121	htmlTable=htmlTable.concat("<table border=\"1\"><tr><td><h4>All "+scenarios.length+" scenarios</h4></td>\n");
133	for (int i= 0; i < configNames.size(); i++){	122	for (int i= 0; i < configNames.size(); i++){
134	label=configNames.get(i).toString();	123	label=configNames.get(i).toString();
Lines 154-185 Link Here
154	String message=null;	143	String message=null;
155	String configName=configNames.get(i).toString();	144	String configName=configNames.get(i).toString();
156	String aUrl=configName;	145	String aUrl=configName;
		146	double[] resultStats = null;
157	if(status.statusMap.get(configName)!=null){	147	if(status.statusMap.get(configName)!=null){
158	message=status.statusMap.get(configName).toString();	148	Object[] statusInfo = (Object[]) status.statusMap.get(configName);
		149	message = (String) statusInfo[0];
		150	resultStats = (double[]) statusInfo[1];
159	}	151	}
160		152
161	if (status.statusMap.containsKey(configName)){	153	if (status.statusMap.containsKey(configName)){
162	int confidence = ((Integer) status.configStatus.get(configName)).intValue();	154	int confidence = ((Integer) status.configStatus.get(configName)).intValue();
163	String image = Utils.getImage(confidence, status.hasSlowDownExplanation);	155	String image = Utils.getImage(confidence, status.hasSlowDownExplanation);
164	if (aUrl!=null){	156	StringBuffer html = new StringBuffer("\n<td><a ");
165	String html="\n<td><a href=\""+aUrl+"/"+status.name.replace('#', '.').replace(':', '_').replace('\\', '_')	157	if ((confidence & Utils.DEV) == 0 \|\| message.length() == 0){
166	+ ".html"+"\">\n<img border=\"0\" src=\"" + image + "\"/></a></td>";	158	// write deviation with error in table when test pass
167		159	html.append("href=\"");
168	//create message with tooltip text if there is a corresponding message	160	html.append(aUrl);
169	if (message.length() > 0){	161	html.append('/');
170	jsIdCount+=1;	162	html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
171	html="<td><a " +	163	html.append(".html\">\n<img border=\"0\" src=\"");
172	"class=\"tooltipSource\" onMouseover=\"show_element('toolTip"+(jsIdCount)+"')\"" +	164	html.append(image);
173	" onMouseout=\"hide_element('toolTip"+(jsIdCount)+"')\" "+	165	html.append("\"/></a>");
174	"\nhref=\""+aUrl+"/"+status.name.replace('#', '.').replace(':', '_').replace('\\', '_')+".html"+"\">" +	166	} else {
175	"<img border=\"0\" src=\""+image+"\"/>" +	167	// create message with tooltip text including deviation with error plus failure message
176	"\n<span class=\"hidden_tooltip\" id=\"toolTip"+jsIdCount+"\">"+message+"</span></a></td>"+	168	jsIdCount+=1;
177	"";	169	html.append("class=\"tooltipSource\" onMouseover=\"show_element('toolTip");
178	}	170	html.append(jsIdCount);
179	htmlTable=htmlTable.concat(html);	171	html.append("')\" onMouseout=\"hide_element('toolTip");
180	} else{	172	html.append(jsIdCount);
181	htmlTable=htmlTable.concat("<td><img title=\""+message+"\" border=\"0\" src=\""+image+"\"/></td>");	173	html.append("')\" \nhref=\"");
182	}	174	html.append(aUrl);
		175	html.append('/');
		176	html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
		177	html.append(".html\">\n<img border=\"0\" src=\"");
		178	html.append(image);
		179	html.append("\"/>\n<span class=\"hidden_tooltip\" id=\"toolTip");
		180	html.append(jsIdCount);
		181	html.append("\">");
		182	html.append(message);
		183	html.append("</span></a>");
		184	}
		185	html.append(Utils.failureMessage(resultStats, false));
		186	html.append("</td>");
		187	htmlTable=htmlTable.concat(html.toString());
183	}else{	188	}else{
184	htmlTable=htmlTable.concat("<td>n/a</td>");	189	htmlTable=htmlTable.concat("<td>n/a</td>");
185	}	190	}

Lines 52-80 Link Here

(-)src/org/eclipse/test/performance/ui/Utils.java (-106 / +76 lines)
52		52
53	public class Utils {	53	public class Utils {
54		54
55	public final static String TTEST_FAILURE_MESSAGE="There is not enough evidence to reject the null hypothesis at the 90% level (Student's t-test).";	55	public final static double STANDARD_ERROR_THRESHOLD = 0.03; // 3%
56	public final static double STANDARD_ERROR_THRESHOLD = 0.02; // 2%
57	static final NumberFormat PERCENT_FORMAT = NumberFormat.getPercentInstance();	56	static final NumberFormat PERCENT_FORMAT = NumberFormat.getPercentInstance();
58	static {	57	static {
59	PERCENT_FORMAT.setMaximumFractionDigits(1);	58	PERCENT_FORMAT.setMaximumFractionDigits(1);
60	}	59	}
61	public final static String STANDARD_ERROR_THRESHOLD_STRING =PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);	60	public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);
62	public final static String TTEST_MODERATION_MESSAGE="Student's t-test failed at the 90% level but it's moderated with standard error lower than "+STANDARD_ERROR_THRESHOLD_STRING;
63	public final static String STANDARD_ERROR_MESSAGE="Standard error on this test is higher than "+STANDARD_ERROR_THRESHOLD_STRING;	61	public final static String STANDARD_ERROR_MESSAGE="Standard error on this test is higher than "+STANDARD_ERROR_THRESHOLD_STRING;
64	public final static String OK_IMAGE="OK.gif";	62	public final static String OK_IMAGE="OK.gif";
65	public final static String OK_IMAGE_ERR="OK_err.gif";
66	public final static String OK_IMAGE_TTEST="OK_ttest.gif";
67	public final static String OK_IMAGE_WARN="OK_caution.gif";	63	public final static String OK_IMAGE_WARN="OK_caution.gif";
68	public final static String FAIL_IMAGE="FAIL.gif";	64	public final static String FAIL_IMAGE="FAIL.gif";
69	public final static String FAIL_IMAGE_TTEST="FAIL_ttest.gif";
70	public final static String FAIL_IMAGE_ERR="FAIL_err.gif";
71	public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";	65	public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";
72	public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";	66	public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";
73	public final static int OK = 0;	67	public final static int OK = 0;
74	public final static int ERR = 0x1;	68	public final static int ERR = 0x1;
75	public final static int TTEST = 0x2;	69	public final static int TTEST = 0x2;
76	public final static int DEV = 0x4;	70	public final static int DEV = 0x4;
77	private final static int NOT_SIGNIFICANT = ERR \| TTEST;
78		71
79	/**	72	/**
80	* @param dimension	73	* @param dimension
Lines 139-151 Link Here
139	}	132	}
140		133
141	/**	134	/**
142	* @return <html><head><meta http-equiv="Content-Type"	135	* Return <html><head><meta http-equiv="Content-Type"
143	* content="text/html; charset=iso-8859-1">	136	* content="text/html; charset=iso-8859-1">
144	*/	137	*/
145	public static String HTML_OPEN = "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\">";	138	public static String HTML_OPEN = "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\">";
146		139
147	/**	140	/**
148	* @return "</html>".	141	* Return "</html>".
149	*/	142	*/
150	public static String HTML_CLOSE = "</html>";	143	public static String HTML_CLOSE = "</html>";
151		144
Lines 170-176 Link Here
170	String description;	163	String description;
171		164
172	/**	165	/**
173	*	166	*
174	* @param name	167	* @param name
175	* the value specifed for the key config in the	168	* the value specifed for the key config in the
176	* eclipse.perf.config system.property key value listings.	169	* eclipse.perf.config system.property key value listings.
Lines 183-195 Link Here
183	this.name = name;	176	this.name = name;
184	this.description = description;	177	this.description = description;
185	}	178	}
186		179
187	public String getName() {	180	public String getName() {
188	return name;	181	return name;
189	}	182	}
190		183
191	public boolean descriptionMatches(String description) {	184	public boolean descriptionMatches(String descr) {
192	return description.equals(this.description);	185	return descr.equals(this.description);
193	}	186	}
194	}	187	}
195		188
Lines 218-224 Link Here
218	/**	211	/**
219	* Queries database with variation composed of buildIdPattern, config and	212	* Queries database with variation composed of buildIdPattern, config and
220	* jvm.	213	* jvm.
221	*	214	*
222	* @return Array of scenarios.	215	* @return Array of scenarios.
223	*/	216	*/
224	public static Scenario[] getScenarios(String buildIdPattern, String scenarioPattern, String config, String jvm) {	217	public static Scenario[] getScenarios(String buildIdPattern, String scenarioPattern, String config, String jvm) {
Lines 232-238 Link Here
232		225
233	/**	226	/**
234	* Creates a Variations object using build id pattern, config and jvm.	227	* Creates a Variations object using build id pattern, config and jvm.
235	*	228	*
236	* @param buildIdPattern	229	* @param buildIdPattern
237	* @param config	230	* @param config
238	* @param jvm	231	* @param jvm
Lines 291-303 Link Here
291		284
292	/**	285	/**
293	* Utility method to copy a file.	286	* Utility method to copy a file.
294	*	287	*
295	* @param src -	288	* @param src the source file.
296	* the source file.	289	* @param dest the destination.
297	* @param dest -
298	* the destination.
299	*/	290	*/
300	public static void copyFile(File src, String dest) {	291	private static void copyFile(File src, File dest) {
301		292
302	try {	293	try {
303	InputStream in = new FileInputStream(src);	294	InputStream in = new FileInputStream(src);
Lines 316-326 Link Here
316	e.printStackTrace();	307	e.printStackTrace();
317	}	308	}
318	}	309	}
		310	public static void copyImages(File images, File output) {
		311	copyFile(new File(images, Utils.FAIL_IMAGE), new File(output, Utils.FAIL_IMAGE));
		312	copyFile(new File(images, Utils.FAIL_IMAGE_EXPLAINED), new File(output, Utils.FAIL_IMAGE_EXPLAINED));
		313	copyFile(new File(images, Utils.FAIL_IMAGE_WARN), new File(output, Utils.FAIL_IMAGE_WARN));
		314	copyFile(new File(images, Utils.OK_IMAGE), new File(output, Utils.OK_IMAGE));
		315	copyFile(new File(images, Utils.OK_IMAGE_WARN), new File(output, Utils.OK_IMAGE_WARN));
		316	}
		317	public static void copyScripts(File scripts, File output) {
		318	copyFile(new File(scripts, "ToolTip.css"), new File(output, "ToolTip.css"));
		319	copyFile(new File(scripts, "ToolTip.js"), new File(output, "ToolTip.js"));
		320	}
319		321
320	/**	322	/**
321	* Returns a LineGraph object representing measurements for a scenario over	323	* Returns a LineGraph object representing measurements for a scenario over
322	* builds.	324	* builds.
323	*	325	*
324	* @param t -	326	* @param t -
325	* the scenario object.	327	* the scenario object.
326	* @param dimensionName -	328	* @param dimensionName -
Lines 368-374 Link Here
368	Color color = black;	370	Color color = black;
369	if (buildID.startsWith("N"))	371	if (buildID.startsWith("N"))
370	color = yellow;	372	color = yellow;
371		373
372	graph.addItem("main", label, dim.getDisplayValue(value), value, color, true, getDateFromBuildID(buildID), true);	374	graph.addItem("main", label, dim.getDisplayValue(value), value, color, true, getDateFromBuildID(buildID), true);
373	continue;	375	continue;
374	}	376	}
Lines 392-398 Link Here
392	continue;	394	continue;
393	} else if (buildID.startsWith("N"))	395	} else if (buildID.startsWith("N"))
394	continue;	396	continue;
395		397
396	for (int i=0;i<currentBuildIdPrefixes.size();i++){	398	for (int i=0;i<currentBuildIdPrefixes.size();i++){
397	if (buildID.startsWith(currentBuildIdPrefixes.get(i).toString())&&!currentFound) {	399	if (buildID.startsWith(currentBuildIdPrefixes.get(i).toString())&&!currentFound) {
398	graph.addItem("main", buildID, dim.getDisplayValue(value), value, black, false, getDateFromBuildID(buildID), false);	400	graph.addItem("main", buildID, dim.getDisplayValue(value), value, black, false, getDateFromBuildID(buildID), false);
Lines 410-416 Link Here
410		412
411	/**	413	/**
412	* Prints a LineGraph object as a gif	414	* Prints a LineGraph object as a gif
413	*	415	*
414	* @param p -	416	* @param p -
415	* the LineGraph object.	417	* the LineGraph object.
416	* @param output -	418	* @param output -
Lines 450-456 Link Here
450	/**	452	/**
451	* Utility method which returns HTML code representing an image map for a	453	* Utility method which returns HTML code representing an image map for a
452	* LineGraph object.	454	* LineGraph object.
453	*	455	*
454	* @param p -	456	* @param p -
455	* the LineGraph object.	457	* the LineGraph object.
456	* @param imageSource -	458	* @param imageSource -
Lines 470-476 Link Here
470		472
471	/**	473	/**
472	* A utility which returns the index of a given buildId in an array.	474	* A utility which returns the index of a given buildId in an array.
473	*	475	*
474	* @param timeSeriesLabels -	476	* @param timeSeriesLabels -
475	* array of buildIds	477	* array of buildIds
476	* @param buildId -	478	* @param buildId -
Lines 586-592 Link Here
586		588
587	/**	589	/**
588	* Returns the date/time from the build id in format yyyymmddhm	590	* Returns the date/time from the build id in format yyyymmddhm
589	*	591	*
590	* @param buildId	592	* @param buildId
591	* @return date/time in format YYYYMMDDHHMM, ie. 200504060010	593	* @return date/time in format YYYYMMDDHHMM, ie. 200504060010
592	*/	594	*/
Lines 650-662 Link Here
650	out = new PrintWriter(new FileWriter(new File(outputFile)));	652	out = new PrintWriter(new FileWriter(new File(outputFile)));
651	out.println(Utils.HTML_OPEN + "</head><body>\n");	653	out.println(Utils.HTML_OPEN + "</head><body>\n");
652	out.println("<h3>Summary of Elapsed Process Variation Coefficients</h3>\n"+	654	out.println("<h3>Summary of Elapsed Process Variation Coefficients</h3>\n"+
653	"<p> This table provides a bird's eye view of variability in elapsed process times\n"+	655	"<p> This table provides a bird's eye view of variability in elapsed process times\n"+
654	"for baseline and current build stream performance scenarios." +	656	"for baseline and current build stream performance scenarios." +
655	" This summary is provided to facilitate the identification of scenarios that should be examined due to high variability." +	657	" This summary is provided to facilitate the identification of scenarios that should be examined due to high variability." +
656	"The variability for each scenario is expressed as a <a href=\"http://en.wikipedia.org/wiki/Coefficient_of_variation\">coefficient\n"+	658	"The variability for each scenario is expressed as a <a href=\"http://en.wikipedia.org/wiki/Coefficient_of_variation\">coefficient\n"+
657	"of variation</a> (CV). The CV is calculated by dividing the <b>standard deviation\n"+	659	"of variation</a> (CV). The CV is calculated by dividing the <b>standard deviation\n"+
658	"of the elapse process time over builds</b> by the <b>average elapsed process\n"+	660	"of the elapse process time over builds</b> by the <b>average elapsed process\n"+
659	"time over builds</b> and multiplying by 100.\n"+	661	"time over builds</b> and multiplying by 100.\n"+
660	"</p><p>High CV values may be indicative of any of the following:<br></p>\n"+	662	"</p><p>High CV values may be indicative of any of the following:<br></p>\n"+
661	"<ol><li> an unstable performance test. </li>\n"+	663	"<ol><li> an unstable performance test. </li>\n"+
662	"<ul><li>may be evidenced by an erratic elapsed process line graph.<br><br></li></ul>\n"+	664	"<ul><li>may be evidenced by an erratic elapsed process line graph.<br><br></li></ul>\n"+
Lines 665-692 Link Here
665	"<li>unstable testing hardware.\n" +	667	"<li>unstable testing hardware.\n" +
666	"<ul><li>consistent higher CV values for one test configuration as compared to others across" +	668	"<ul><li>consistent higher CV values for one test configuration as compared to others across" +
667	" scenarios may be related to hardward problems.</li></ul></li></ol>\n"+	669	" scenarios may be related to hardward problems.</li></ul></li></ol>\n"+
668	"<p> Scenarios are listed in alphabetical order in the far right column. A scenario's\n"+	670	"<p> Scenarios are listed in alphabetical order in the far right column. A scenario's\n"+
669	"variation coefficients (CVs) are in columns to the left for baseline and current\n"+	671	"variation coefficients (CVs) are in columns to the left for baseline and current\n"+
670	"build streams for each test configuration. Scenarios with CVs > 10% are highlighted\n"+	672	"build streams for each test configuration. Scenarios with CVs > 10% are highlighted\n"+
671	"in yellow (10%<CV><CV<20%) and orange(CV>20%). </p>\n"+	673	"in yellow (10%<CV><CV<20%) and orange(CV>20%). </p>\n"+
672	"<p> Each CV value links to the scenario's detailed results to allow viewers to\n"+	674	"<p> Each CV value links to the scenario's detailed results to allow viewers to\n"+
673	"investigate the variability.</p>\n");	675	"investigate the variability.</p>\n");
674		676
675	Hashtable cvTable = (Hashtable) rawDataTables.get(scenarios[0]);	677	Hashtable cvTable = (Hashtable) rawDataTables.get(scenarios[0]);
676	String[] configNames = (String[]) cvTable.keySet().toArray(new String[cvTable.size()]);	678	String[] configNames = (String[]) cvTable.keySet().toArray(new String[cvTable.size()]);
677	Arrays.sort(configNames);	679	Arrays.sort(configNames);
678		680
679		681
680	int configColumns=configNames.length/2;	682	int configColumns=configNames.length/2;
681	out.println("<table border=\"1\"><tr>" +	683	out.println("<table border=\"1\"><tr>" +
682	"<td colspan=\""+configColumns+"\"><b>Baseline CVs</b></td>"+	684	"<td colspan=\""+configColumns+"\"><b>Baseline CVs</b></td>"+
683	"<td colspan=\""+configColumns+"\"><b>Current Build Stream CVs</b></td>"+	685	"<td colspan=\""+configColumns+"\"><b>Current Build Stream CVs</b></td>"+
684	"<td rowspan=\"2\"><b>Scenario Name</b></td>"+	686	"<td rowspan=\"2\"><b>Scenario Name</b></td>"+
685	"</tr><tr>");	687	"</tr><tr>");
686		688
687		689
688	for (int i = 0; i < configNames.length; i++) {	690	for (int i = 0; i < configNames.length; i++) {
689	//configNames here have prefix cConfig- or bConfig- depending on whether the data comes from	691	//configNames here have prefix cConfig- or bConfig- depending on whether the data comes from
690	//current build stream data or baseline data.	692	//current build stream data or baseline data.
691	out.print("<td>" + ((ConfigDescriptor)configDescriptors.get(configNames[i].substring(8))).description + "</td>");	693	out.print("<td>" + ((ConfigDescriptor)configDescriptors.get(configNames[i].substring(8))).description + "</td>");
692	}	694	}
Lines 696-702 Link Here
696	Hashtable aCvTable = (Hashtable) rawDataTables.get(scenarios[i]);	698	Hashtable aCvTable = (Hashtable) rawDataTables.get(scenarios[i]);
697	String scenario = scenarios[i];	699	String scenario = scenarios[i];
698	String scenarioFile=scenario.replace('#', '.').replace(':', '_').replace('\\', '_')+".html";	700	String scenarioFile=scenario.replace('#', '.').replace(':', '_').replace('\\', '_')+".html";
699		701
700	for (int j = 0; j < configNames.length; j++) {	702	for (int j = 0; j < configNames.length; j++) {
701	String url=configNames[j].substring(8)+"/"+scenarioFile;	703	String url=configNames[j].substring(8)+"/"+scenarioFile;
702	if (aCvTable.get(configNames[j]) == null) {	704	if (aCvTable.get(configNames[j]) == null) {
Lines 776-816 Link Here
776		778
777	public static boolean hasConfidentResult(Variations variations, String scenarioName, String baseline, String config) {	779	public static boolean hasConfidentResult(Variations variations, String scenarioName, String baseline, String config) {
778	double[] resultStats = resultStats(variations, scenarioName, baseline, config);	780	double[] resultStats = resultStats(variations, scenarioName, baseline, config);
779	if (resultStats != null) {	781	return (confidenceLevel(resultStats) & ERR) == 0;
780	return resultStats[1] < 0 \|\| resultStats[0] < resultStats[1] \|\| resultStats[2] < STANDARD_ERROR_THRESHOLD;
781	}
782	return true;
783	}	782	}
784	public static String failureMessage(Variations variations, String scenarioName, String baseline, String config, boolean prefix) {	783	public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) {
785	return failureMessage(resultStats(variations, scenarioName, baseline, config), prefix);	784	return failureMessage(resultStats(variations, scenarioName, baseline, config), true);
786	}	785	}
787	public static String failureMessage(double[] resultStats, boolean prefix) {	786	public static String failureMessage(double[] resultStats, boolean full) {
788	StringBuffer buffer = null;	787	StringBuffer buffer = new StringBuffer();
789	switch (confidenceLevel(resultStats)) {	788	if (full & (confidenceLevel(resultStats) & ERR) != 0) {
790	case ERR:	789	buffer.append("* WARNING * ");
791	buffer = new StringBuffer();	790	buffer.append(STANDARD_ERROR_MESSAGE);
792	if (prefix) buffer.append("* INFO * ");	791	}
793	buffer.append(STANDARD_ERROR_MESSAGE);	792	if (!full) buffer.append("<small><font color=\"gray\"> ");
794	buffer.append(" (");	793	if (resultStats == null) {
795	buffer.append(PERCENT_FORMAT.format(resultStats[2]));	794	buffer.append('?');
796	buffer.append(")");	795	} else {
797	break;	796	double deviation = resultStats[3]==0 ? 0 : -resultStats[3];
798	case TTEST:	797	if (deviation > 0) {
799	buffer = new StringBuffer();	798	buffer.append('+');
800	if (prefix) buffer.append("* INFO * ");	799	}
801	buffer.append(TTEST_MODERATION_MESSAGE);	800	buffer.append(PERCENT_FORMAT.format(deviation));
802	buffer.append(" (");	801	buffer.append(" (±");
803	buffer.append(PERCENT_FORMAT.format(resultStats[2]));	802	buffer.append(PERCENT_FORMAT.format(resultStats[2]));
804	buffer.append(")");	803	buffer.append(")");
805	break;
806	case TTEST\|ERR:
807	buffer = new StringBuffer();
808	if (prefix) buffer.append("* WARNING * ");
809	buffer.append(TTEST_FAILURE_MESSAGE);
810	break;
811	default:
812	return null;
813	}	804	}
		805	if (!full) buffer.append("</font></small>");
814	return buffer.toString();	806	return buffer.toString();
815	}	807	}
816	public static int confidenceLevel(double[] resultStats) {	808	public static int confidenceLevel(double[] resultStats) {
Lines 860-900 Link Here
860	public static String getImage(int confidence, boolean hasExplanation) {	852	public static String getImage(int confidence, boolean hasExplanation) {
861	boolean scenarioFailed = (confidence & DEV) != 0;	853	boolean scenarioFailed = (confidence & DEV) != 0;
862	String image = null;	854	String image = null;
863		855
864	if (scenarioFailed) {	856	if (scenarioFailed) {
865	if (hasExplanation) {	857	if (hasExplanation) {
866	image = FAIL_IMAGE_EXPLAINED;	858	image = FAIL_IMAGE_EXPLAINED;
		859	} else if ((confidence & ERR) != 0) {
		860	image = FAIL_IMAGE_WARN;
867	} else {	861	} else {
868	switch (confidence) {	862	image = FAIL_IMAGE;
869	default:
870	image = FAIL_IMAGE;
871	break;
872	case TTEST:
873	image = FAIL_IMAGE_TTEST;
874	break;
875	case ERR:
876	image = FAIL_IMAGE_ERR;
877	break;
878	case NOT_SIGNIFICANT:
879	image = FAIL_IMAGE_WARN;
880	break;
881	}
882	}	863	}
883	} else {	864	} else if ((confidence & ERR) != 0) {
884	switch (confidence) {	865	image = OK_IMAGE_WARN;
885	default:	866	} else {
886	image = OK_IMAGE;	867	image = OK_IMAGE;
887	break;
888	case TTEST:
889	image = OK_IMAGE_TTEST;
890	break;
891	case ERR:
892	image = OK_IMAGE_ERR;
893	break;
894	case NOT_SIGNIFICANT:
895	image = OK_IMAGE_WARN;
896	break;
897	}
898	}	868	}
899	return image;	869	return image;
900	}	870	}

Lines 11-17 Link Here

(-)src/org/eclipse/test/performance/ui/Main.java (-12 / +2 lines)
11	package org.eclipse.test.performance.ui;	11	package org.eclipse.test.performance.ui;
12		12
13	import java.io.File;	13	import java.io.File;
14	import java.io.FileNotFoundException;
15	import java.io.FileOutputStream;	14	import java.io.FileOutputStream;
16	import java.io.PrintStream;	15	import java.io.PrintStream;
17	import java.net.URL;	16	import java.net.URL;
Lines 68-87 Link Here
68		67
69	if (images!=null) {	68	if (images!=null) {
70	images= Platform.resolve(images);	69	images= Platform.resolve(images);
71	Utils.copyFile(new File(images.getPath(), "FAIL.gif"), output + "/FAIL.gif");	70	Utils.copyImages(new File(images.getPath()), new File(output));
72	Utils.copyFile(new File(images.getPath(), "FAIL_ttest.gif"), output + "/FAIL_ttest.gif");
73	Utils.copyFile(new File(images.getPath(), "FAIL_err.gif"), output + "/FAIL_err.gif");
74	Utils.copyFile(new File(images.getPath(), "FAIL_greyed.gif"), output+"/FAIL_greyed.gif");
75	Utils.copyFile(new File(images.getPath(), "FAIL_caution.gif"), output+"/FAIL_caution.gif");
76	Utils.copyFile(new File(images.getPath(), "OK.gif"),output + "/OK.gif");
77	Utils.copyFile(new File(images.getPath(), "OK_err.gif"),output + "/OK_err.gif");
78	Utils.copyFile(new File(images.getPath(), "OK_ttest.gif"),output + "/OK_ttest.gif");
79	Utils.copyFile(new File(images.getPath(), "OK_caution.gif"),output + "/OK_caution.gif");
80	}	71	}
81	if (scripts!=null){	72	if (scripts!=null){
82	scripts= Platform.resolve(scripts);	73	scripts= Platform.resolve(scripts);
83	Utils.copyFile(new File(scripts.getPath(), "ToolTip.css"),output+"/ToolTip.css");	74	Utils.copyScripts(new File(scripts.getPath()), new File(output));
84	Utils.copyFile(new File(scripts.getPath(), "ToolTip.js"),output+"/ToolTip.js");
85	}	75	}
86		76
87	// print fingerprint/scenario status pages	77	// print fingerprint/scenario status pages

Lines 83-117 Link Here

(-)src/org/eclipse/test/internal/performance/eval/StatisticsUtil.java (-17 / +12 lines)
83	* XXX the current implementation assumes that the standard deviations are sufficiently similar.	83	* XXX the current implementation assumes that the standard deviations are sufficiently similar.
84	* </p>	84	* </p>
85	*	85	*
86	* @param series1 the time series containing the first data set	86	* @param refSeries the time series containing the first data set
87	* @param index1 the index into <code>series1</code> for the first data set	87	* @param index1 the index into <code>series1</code> for the first data set
88	* @param series2 the time series containing the second data set	88	* @param testSeries the time series containing the second data set
89	* @param index2 the index into <code>series2</code> for the second data set	89	* @param index2 the index into <code>series2</code> for the second data set
90	* @param percentile the percentile level to use	90	* @param percentile the percentile level to use
91	* @return <code>true</code> if the null hypothesis is rejected on the <code>percentile</code>	91	* @return <code>true</code> if the null hypothesis is rejected on the <code>percentile</code>
92	* level, <code>false</code> if it cannot be rejected based on the given data	92	* level, <code>false</code> if it cannot be rejected based on the given data
93	*/	93	*/
94	public static boolean hasSignificantDifference(TimeSeries series1, int index1, TimeSeries series2, int index2, Percentile percentile) {	94	public static double[] statisticsForTimeSeries(TimeSeries refSeries, int index1, TimeSeries testSeries, int index2, Percentile percentile) {
95	// see http://bmj.bmjjournals.com/collections/statsbk/7.shtml	95	// see http://bmj.bmjjournals.com/collections/statsbk/7.shtml
96		96
97	double[] values = new double[] { series1.getValue(index1), series2.getValue(index2) };	97	double[] values = new double[] { refSeries.getValue(index1), testSeries.getValue(index2) };
98	long[] counts = new long[] { series1.getCount(index1), series2.getCount(index2) };	98	long[] counts = new long[] { refSeries.getCount(index1), testSeries.getCount(index2) };
99	double[] stddevs = new double[] { series1.getStddev(index1), series2.getStddev(index2) };	99	double[] stddevs = new double[] { refSeries.getStddev(index1), testSeries.getStddev(index2) };
100	double t = studentTtest(values, stddevs, counts, percentile);
101	return t == -1 \|\| getStudentsT((int) (counts[0] + counts[1] - 2), percentile) < t;
102	}
103
104	public static double[] statisticsForTimeSeries(TimeSeries series1, int index1, TimeSeries series2, int index2, Percentile percentile) {
105	// see http://bmj.bmjjournals.com/collections/statsbk/7.shtml
106
107	double[] values = new double[] { series1.getValue(index1), series2.getValue(index2) };
108	long[] counts = new long[] { series1.getCount(index1), series2.getCount(index2) };
109	double[] stddevs = new double[] { series1.getStddev(index1), series2.getStddev(index2) };
110	double ttest = studentTtest(values, stddevs, counts, percentile);	100	double ttest = studentTtest(values, stddevs, counts, percentile);
111	return new double[] {	101	return new double[] {
112	getStudentsT((int) (counts[0] + counts[1] - 2), percentile),	102	getStudentsT((int) (counts[0] + counts[1] - 2), percentile),
113	ttest,	103	ttest,
114	standardError(values, stddevs, counts),	104	standardError(values, stddevs, counts),
		105	deviation(values),
115	};	106	};
116	}	107	}
117		108
Lines 138-146 Link Here
138	return -1;	129	return -1;
139	}	130	}
140		131
		132	public static double deviation(double[] values) {
		133	return (values[1] - values[0]) / values[0];
		134	}
		135
141	public static double standardError(double[] values, double[] stddevs, long[] counts) {	136	public static double standardError(double[] values, double[] stddevs, long[] counts) {
142	return Math.sqrt((stddevs[0] * stddevs[0] / counts[0]) + (stddevs[1] * stddevs[1] / counts[1])) / values[0];	137	return Math.sqrt((stddevs[0] * stddevs[0] / counts[0]) + (stddevs[1] * stddevs[1] / counts[1])) / values[0];
143	}	138	}
144		139
145	/**	140	/**
146	* The (two-tailed) T-table. [degrees_of_freedom][percentile]	141	* The (two-tailed) T-table. [degrees_of_freedom][percentile]