Attachment #70131 for bug #185580




import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Map;
import java.util.StringTokenizer;

import junit.framework.AssertionFailedError;

import org.eclipse.test.internal.performance.data.Dim;
import org.eclipse.test.internal.performance.db.DB;
import org.eclipse.test.internal.performance.db.Scenario;
import org.eclipse.test.internal.performance.db.SummaryEntry;
import org.eclipse.test.internal.performance.db.TimeSeries;
import org.eclipse.test.internal.performance.db.Variations;
import org.eclipse.test.internal.performance.eval.StatisticsUtil;
import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
import org.eclipse.test.performance.Dimension;



	static {
		PERCENT_FORMAT.setMaximumFractionDigits(1);
	}
	static final DecimalFormat DEVIATION_FORMAT = (DecimalFormat) NumberFormat.getPercentInstance();
	static {
		DEVIATION_FORMAT.setMaximumFractionDigits(1);
		DEVIATION_FORMAT.setMinimumFractionDigits(1);
		DEVIATION_FORMAT.setPositivePrefix("+");
		DEVIATION_FORMAT.setNegativePrefix("- ");
	}
	static final DecimalFormat STDERR_FORMAT = (DecimalFormat) NumberFormat.getNumberInstance();
	static {
		STDERR_FORMAT.setMaximumFractionDigits(1);
		STDERR_FORMAT.setMinimumFractionDigits(1);
		STDERR_FORMAT.setMultiplier(100);
	}
	public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);
	public final static String UNKNOWN_IMAGE="Unknown.gif";
	public final static String OK_IMAGE="OK.gif";
	public final static String OK_IMAGE_WARN="OK_caution.gif";
	public final static String FAIL_IMAGE="FAIL.gif";
	public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";
	public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";
	public final static int OK = 0;
	public final static int NAN = 0x1;
	public final static int ERR = 0x2;
//	public final static int TTEST = 0x2;
	public final static int DEV = 0x4;

	/**


		while (tokenizer.hasMoreTokens()) {
			String labelDescriptor = tokenizer.nextToken();
			String[] elements = labelDescriptor.trim().split(",");
			ConfigDescriptor descriptor = new ConfigDescriptor(elements[0], elements[1]);
			configMap.put(elements[0], descriptor);
		}

		return componentNames;
	}

	/*
	 * @param fp -
	 *            a FingerPrint object
	 * @return - an html representation of the fingerprint.
	 *
	public static String getImageMap(FingerPrint fp) {
		String componentDescription = fp.configDescriptor.description;
		String areas = fp.bar.getAreas();

		}
		return output;
	}
	*/

	/**
	 * Utility method to copy a file.

		}
	}
	public static void copyImages(File images, File output) {
		copyFile(new File(images, FAIL_IMAGE), new File(output, FAIL_IMAGE));
		copyFile(new File(images, FAIL_IMAGE_EXPLAINED), new File(output, FAIL_IMAGE_EXPLAINED));
		copyFile(new File(images, FAIL_IMAGE_WARN), new File(output, FAIL_IMAGE_WARN));
		copyFile(new File(images, OK_IMAGE), new File(output, OK_IMAGE));
		copyFile(new File(images, OK_IMAGE_WARN), new File(output, OK_IMAGE_WARN));
		copyFile(new File(images, UNKNOWN_IMAGE), new File(output, UNKNOWN_IMAGE));
	}
	public static void copyScripts(File scripts, File output) {
		copyFile(new File(scripts, "ToolTip.css"), new File(output, "ToolTip.css"));

		p.paint(image);

		/* Downscale to 8 bit depth palette to save to gif */
		ImageData data = downSample(image);
		ImageLoader il = new ImageLoader();
		il.data = new ImageData[] { data };
		OutputStream out = null;

	 * @return date/time in format YYYYMMDDHHMM, ie. 200504060010
	 */
	public static long getDateFromBuildID(String buildId) {
		return getDateFromBuildID(buildId, false);
	}

	public static long getDateFromBuildID(String buildId, boolean matchLast) {

		PrintWriter out=null;
		try {
			out = new PrintWriter(new FileWriter(new File(outputFile)));
			out.println(HTML_OPEN + "</head><body>\n");
			out.println("<h3>Summary of Elapsed Process Variation Coefficients</h3>\n"+
		"<p> This table provides a bird's eye view of variability in elapsed process times\n"+
		  "for baseline and current build stream performance scenarios." +

			out.close();
		}
	}

    public static double[] resultsStatistics(TimeSeries timeSeries) {
    	try {
	    	double valueRef = timeSeries.getValue(0), value = timeSeries.getValue(1);
	    	long countRef = timeSeries.getCount(0), count = timeSeries.getCount(1);
	    	double stddevRef = timeSeries.getStddev(0), stddev = timeSeries.getStddev(1);
	    	double stderr = (countRef == 1 || count == 1)
    			? Double.NaN
				: (Double.isNaN(stddevRef)
					? Math.sqrt((stddev * stddev / count)) / valueRef
					: Math.sqrt((stddevRef * stddevRef / countRef) + (stddev * stddev / count)) / valueRef);
			return new double[] {
				(value - valueRef) / valueRef,
				stderr,
			};
    	}
    	catch (ArrayIndexOutOfBoundsException aioobe) {
    		return null;
    	}
		}

		for (int i = 0; i < currentScenarios.length; i++) {
			Scenario scenario = currentScenarios[i];
			String name = scenario.getScenarioName();
			currentScenariosMap.put(name, scenario);
		}
		Percentile percentile = StatisticsUtil.T90;
		Scenario scenario = (Scenario) currentScenariosMap.get(scenarioName);

		Scenario reference = (Scenario) referenceScenariosMap.get(scenarioName);
		if (reference != null) {
			// XXX have to find out the relevant dimension
			Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;
			TimeSeries currentSeries = scenario.getTimeSeries(significanceDimension);
			TimeSeries baselineSeries = reference.getTimeSeries(significanceDimension);
			if (currentSeries.getLength() > 0 && baselineSeries.getLength() > 0) {
				return StatisticsUtil.statisticsForTimeSeries(baselineSeries, 0, currentSeries, 0, percentile);
			}
		}
		return null;
	}

	public static boolean hasConfidentResult(TimeSeries timeSeries) {
	    double[] resultStats = resultsStatistics(timeSeries);
	    return (confidenceLevel(resultStats) & ERR) == 0;
    }
	public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) {
		String current = (String) variations.get(PerformanceTestPlugin.BUILD);
		Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;
		Scenario newScenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, baseline, current, new Dim[] { significanceDimension });
        TimeSeries timeSeries = newScenario.getTimeSeries(significanceDimension);
        double[] results = resultsStatistics(timeSeries);
		return failureMessage(results, true);
	}
	public static String failureMessage(double[] resultStats, boolean full) {
		StringBuffer buffer = new StringBuffer();
		int level = confidenceLevel(resultStats);
//		boolean isWarn = (level & WARN) != 0;
		boolean isErr = (level & ERR) != 0;
		if (full) {
			if (isErr) {
				buffer.append("*** WARNING ***  ");
	 			buffer.append(Messages.bind(Messages.standardError, PERCENT_FORMAT.format(resultStats[1]), STANDARD_ERROR_THRESHOLD_STRING));
			}
			return buffer.toString();
		}
		if (!full) buffer.append("<font color=\"#0000FF\" size=\"1\">  ");
		if (resultStats != null) {
			double deviation = resultStats[0];
			buffer.append("<font color=\"#0000FF\" size=\"1\">");
			if (Double.isNaN(deviation) || Double.isInfinite(deviation)) {
	 			buffer.append(" [n/a]");
 			} else {
				double stderr = resultStats[1];
				deviation = Math.abs(deviation)<0.001 ? 0 : -deviation;
	 			if (Double.isNaN(stderr) || Double.isInfinite(stderr)) {
		 			buffer.append(DEVIATION_FORMAT.format(deviation));
					buffer.append("</font><font color=\"#DDDD00\" size=\"1\"> ");
		 			buffer.append(" [n/a]");
	 			} else {
		 			buffer.append(DEVIATION_FORMAT.format(deviation));
	 				buffer.append(" [&#177;");
	 				buffer.append(STDERR_FORMAT.format(Math.abs(stderr)));
	 				buffer.append(']');
	 			}
 			}
			buffer.append("</font>");
		}
		if (!full) buffer.append("</font>");
		return buffer.toString();
	}
	public static int confidenceLevel(double[] resultStats) {
		int level = OK;
 		if (resultStats != null){
			if (Double.isNaN(resultStats[0]) || Double.isInfinite(resultStats[0])) {
				level = NAN;
 			} else {
//	 			if (resultStats[1] >= (STANDARD_ERROR_THRESHOLD/2)) { // warns standard error higher than the half of authorized threshold
//	 				level |= WARN;
//	 			}
	 			if (resultStats[1] >= STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold
	 				level = ERR;
	 			}
 			}
 		}
		return level;

			String previous = "";
			while (tokenizer.hasMoreTokens()) {
				String token = tokenizer.nextToken();
				if (!token.equals("%") && !token.equals("_")) {
					start += previous.length();
				} else if (token.equals("_")) {
					start++;
				} else {
					if (previous.equals("%")) {
						int idx = name.substring(start).indexOf(token);
						if (idx < 0) return false;
						start += idx;
					} else if (previous.equals("_")) {
						if (!name.substring(++start).startsWith(token)) return false;
					}
					start += token.length();
				}

		return name.equals(pattern);
	}

	public static String getImage(int confidence, boolean hasExplanation) {
	    boolean scenarioFailed = (confidence & DEV) != 0;
	    String image = null;


		    } else {
    			image = FAIL_IMAGE;
		    }
	    } else if ((confidence & NAN) != 0) {
			image = UNKNOWN_IMAGE;
	    } else if ((confidence & ERR) != 0) {
	   		image = OK_IMAGE_WARN;
	    } else {

	    }
	    return image;
    }

	public static boolean hasSummary(SummaryEntry[] summaries, String scenarioName) {
		int length = summaries == null ? 0 : summaries.length;
		for (int i=0; i<length; i++) {
			SummaryEntry summary = summaries[i];
			if (summary.scenarioName.equals(scenarioName)) {
				return true;
			}
		}
		return false;
	}

	public static String getScenarioShortName(String scenarioName, int max) {

		// Remove class name qualification
		int testSeparator = scenarioName.indexOf('#');
		boolean hasClassName = testSeparator >= 0;
		if (!hasClassName) {
			testSeparator = scenarioName.lastIndexOf('.');
			if (testSeparator <= 0) {
				if (max > 0 && scenarioName.length() > max) {
					return "*"+scenarioName.substring(0, max);
				}
				return scenarioName;
			}
		}
		int classSeparator = scenarioName.substring(0, testSeparator).lastIndexOf('.');
		if (classSeparator < 0) {
			if (max > 0 && scenarioName.length() > max) {
				return "*"+scenarioName.substring(0, max);
			}
			return scenarioName;
		}
		int length = scenarioName.length();
		String shortName = scenarioName.substring(classSeparator+1, length);
		if (!hasClassName && shortName.startsWith("test.")) { // specific case for swt...
			shortName = shortName.substring(5);
		}

		// Remove qualification from test name
		StringTokenizer tokenizer = new StringTokenizer(shortName, " :,", true);
		StringBuffer buffer = new StringBuffer(tokenizer.nextToken());
		while (tokenizer.hasMoreTokens()) {
			String token = tokenizer.nextToken();
			char fc = token.charAt(0);
			while (fc == ' ' || fc == ',' || fc == ':') {
				buffer.append(token); // add the separator
				token = tokenizer.nextToken();
				fc = token.charAt(0);
			}
			int last = token .lastIndexOf('.');
			if (last >= 3) {
				int first = token .indexOf('.');
				if (first == last) {
					buffer.append(token);
				} else {
//					buffer.append(token.substring(0, first));
//					buffer.append("...");
					buffer.append(token.substring(last+1));
				}
			} else {
				buffer.append(token);
			}
		}
		if (max > 0 && buffer.length() > max) {
			return "*"+buffer.substring(0, max);
		}
		return buffer.toString();
	}
}

Lines 10-21 Link Here

(-)src/org/eclipse/test/performance/ui/ScenarioStatusTable.java (-86 / +181 lines)
10	*******************************************************************************/	10	*******************************************************************************/
11	package org.eclipse.test.performance.ui;	11	package org.eclipse.test.performance.ui;
12		12
		13	import java.io.PrintStream;
13	import java.util.ArrayList;	14	import java.util.ArrayList;
14	import java.util.Hashtable;	15	import java.util.Hashtable;
		16	import java.util.StringTokenizer;
15		17
		18	import org.eclipse.test.internal.performance.PerformanceTestPlugin;
		19	import org.eclipse.test.internal.performance.data.Dim;
16	import org.eclipse.test.internal.performance.db.DB;	20	import org.eclipse.test.internal.performance.db.DB;
17	import org.eclipse.test.internal.performance.db.Scenario;	21	import org.eclipse.test.internal.performance.db.Scenario;
		22	import org.eclipse.test.internal.performance.db.SummaryEntry;
		23	import org.eclipse.test.internal.performance.db.TimeSeries;
18	import org.eclipse.test.internal.performance.db.Variations;	24	import org.eclipse.test.internal.performance.db.Variations;
		25	import org.eclipse.test.performance.Dimension;
19		26
20	public class ScenarioStatusTable {	27	public class ScenarioStatusTable {
21		28
Lines 24-41 Link Here
24	private String scenarioPattern;	31	private String scenarioPattern;
25	private ArrayList configNames=new ArrayList();	32	private ArrayList configNames=new ArrayList();
26	private Hashtable scenarioComments;	33	private Hashtable scenarioComments;
		34	private SummaryEntry[] fingerprintEntries;
27	private final String baseline;	35	private final String baseline;
28		36
29	private class ScenarioStatus{	37	private class ScenarioStatus{
30	Hashtable statusMap;	38	Hashtable statusMap;
31	String name;	39	String name, shortName;
32	Hashtable configStatus;	40	Hashtable configStatus;
		41	Hashtable resultsMap;
33	boolean hasSlowDownExplanation=false;	42	boolean hasSlowDownExplanation=false;
		43	boolean fingerprint = false;
		44	boolean hasBaseline = true;
34		45
35	public ScenarioStatus(String scenarioName){	46	public ScenarioStatus(String scenarioName){
36	name=scenarioName;	47	name = scenarioName;
37	statusMap=new Hashtable();	48	statusMap = new Hashtable();
38	configStatus=new Hashtable();	49	configStatus = new Hashtable();
		50	resultsMap = new Hashtable();
39	}	51	}
40		52
41	}	53	}
Lines 48-67 Link Here
48	* @param scenarioPattern	60	* @param scenarioPattern
49	* @param configDescriptors	61	* @param configDescriptors
50	*/	62	*/
51	public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, String baseline){	63	public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, SummaryEntry[] fpSummaries, String baseline){
52	configMaps=configDescriptors;	64	configMaps=configDescriptors;
53	this.variations=variations;	65	this.variations=variations;
54	this.scenarioPattern=scenarioPattern;	66	this.scenarioPattern=scenarioPattern;
55	this.scenarioComments=scenarioComments;	67	this.scenarioComments=scenarioComments;
56	this.baseline= baseline;	68	this.baseline= baseline;
		69	this.fingerprintEntries = fpSummaries;
57	}	70	}
58		71
59	/**	72	/**
60	* Returns HTML representation of scenario status table.	73	* Prints the HTML representation of scenario status table into the given stream.
61	*/	74	*/
62	public String toString() {	75	public void print(PrintStream stream, boolean filter) {
63	String OS="config";	76	String OS="config";
64	String htmlTable="";
65	Scenario[] scenarios= DB.queryScenarios(variations, scenarioPattern, OS, null);	77	Scenario[] scenarios= DB.queryScenarios(variations, scenarioPattern, OS, null);
66		78
67	if (scenarios != null && scenarios.length > 0) {	79	if (scenarios != null && scenarios.length > 0) {
Lines 70-94 Link Here
70	for (int i= 0; i < scenarios.length; i++) {	82	for (int i= 0; i < scenarios.length; i++) {
71	Scenario scenario= scenarios[i];	83	Scenario scenario= scenarios[i];
72	String scenarioName=scenario.getScenarioName();	84	String scenarioName=scenario.getScenarioName();
73	// if (!Utils.matchPattern(scenarioName, scenarioPattern)) continue;	85	if (filter && !Utils.matchPattern(scenarioName, scenarioPattern)) continue;
74	// returns the config names. Making assumption that indices in	86	// returns the config names. Making assumption that indices in
75	// the configs array map to the indices of the failure messages.	87	// the configs array map to the indices of the failure messages.
76	String[] configs=scenario.getTimeSeriesLabels();	88	String[] configs=scenario.getTimeSeriesLabels();
77	String[] failureMessages= scenario.getFailureMessages();	89	String[] failureMessages= scenario.getFailureMessages();
78	ScenarioStatus scenarioStatus=new ScenarioStatus(scenarioName);	90	ScenarioStatus scenarioStatus=new ScenarioStatus(scenarioName);
79	scenarioStatusList.add(scenarioStatus);	91	scenarioStatus.fingerprint = Utils.hasSummary(this.fingerprintEntries, scenarioName);
80		92
81	String scenarioComment= (String)scenarioComments.get(scenarioName);	93	String scenarioComment= (String)scenarioComments.get(scenarioName);
82	if (scenarioComment != null)	94	if (scenarioComment != null)
83	scenarioStatus.hasSlowDownExplanation= true;	95	scenarioStatus.hasSlowDownExplanation= true;
84		96
85	for (int j=0;j<configs.length;j++){	97	int confsLength = configs.length;
		98	for (int j=0; j<confsLength; j++){
86	if (!configNames.contains(configs[j]))	99	if (!configNames.contains(configs[j]))
87	configNames.add(configs[j]);	100	configNames.add(configs[j]);
88		101
89	double[] resultStats = Utils.resultStats(variations, scenarioName, baseline, configs[j]);	102	// Compute confidence level and store it in scenario status
		103	Variations v = (Variations) variations.clone();
		104	v.put(PerformanceTestPlugin.CONFIG, configs[j]);
		105	// double[] resultStats = Utils.resultStats(v, scenarioName, baseline, configs[j]);
		106	String current = (String) v.get(PerformanceTestPlugin.BUILD);
		107	Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;
		108	Scenario newScenario= DB.getScenarioSeries(scenarioName, v, PerformanceTestPlugin.BUILD, baseline, current, new Dim[] { significanceDimension });
		109	String[] timeSeriesLabels= newScenario.getTimeSeriesLabels();
		110	TimeSeries timeSeries = newScenario.getTimeSeries(significanceDimension);
		111	boolean hasBaseline = timeSeriesLabels.length == 2 && timeSeriesLabels[0].equals(baseline);
		112	double[] resultStats = Utils.resultsStatistics(timeSeries);
		113	if (resultStats == null) continue;
		114	if (resultStats != null && resultStats[1] < 0 && scenarioStatus.hasBaseline) scenarioStatus.hasBaseline = false;
90	int confidenceLevel = Utils.confidenceLevel(resultStats);	115	int confidenceLevel = Utils.confidenceLevel(resultStats);
		116	scenarioStatus.configStatus.put(configs[j], new Integer(confidenceLevel));
91		117
		118	// Store failure message in scenario status
92	boolean hasScenarioFailure = failureMessages[j] != null && failureMessages[j].indexOf(configs[j]) != -1; // ensure correct failure message relates to config	119	boolean hasScenarioFailure = failureMessages[j] != null && failureMessages[j].indexOf(configs[j]) != -1; // ensure correct failure message relates to config
93	StringBuffer buffer = new StringBuffer();	120	StringBuffer buffer = new StringBuffer();
94	if (hasScenarioFailure) {	121	if (hasScenarioFailure) {
Lines 99-145 Link Here
99	}	126	}
100	confidenceLevel \|= Utils.DEV;	127	confidenceLevel \|= Utils.DEV;
101	}	128	}
		129	scenarioStatus.statusMap.put(configs[j], buffer.toString());
102		130
103	scenarioStatus.configStatus.put(configs[j], new Integer(confidenceLevel));	131	// Store text for numbers in scenario status
104	scenarioStatus.statusMap.put(configs[j], new Object[] { buffer.toString(), resultStats });	132	String text = Utils.failureMessage(resultStats, false);
		133	scenarioStatus.resultsMap.put(configs[j], text);
		134
		135	// Store scenario short name in scenario status (for table column)
		136	if (scenarioStatus.shortName == null) {
		137	if (hasBaseline) { // baseline is OK
		138	scenarioStatus.shortName = Utils.getScenarioShortName(scenarioName, -1);
		139	} else {
		140	StringBuffer shortName = new StringBuffer("*");
		141	shortName.append(Utils.getScenarioShortName(scenarioName, -1));
		142	shortName.append(" <small>(vs. ");
		143	shortName.append(timeSeriesLabels[0]);
		144	shortName.append(")</small>");
		145	scenarioStatus.shortName = shortName.toString();
		146	}
		147	}
		148
		149	// Store scenario status
		150	if (!scenarioStatusList.contains(scenarioStatus)) {
		151	scenarioStatusList.add(scenarioStatus);
		152	}
105	}	153	}
106	}	154	}
107		155
108	String label=null;	156	String label=null;
109	htmlTable=htmlTable.concat("<br><h4>Scenario Status</h4>\n" +	157	stream.println("<br><h4>Scenario Status</h4>");
110	"The scenario status table shows all scenarios tests result for all performance test machines. It gives a complete but compact view of performance result for the component.<br>\n" +	158	stream.println("The following table gives a complete but compact view of performance results for the component.<br>");
111	"For each test (ie. in each cell of this table), following information are displayed:\n" +	159	stream.println("Each line of the table shows the results for one scenario on all machines.<br><br>");
112	"<ul>\n" +	160	stream.println("The name of the scenario is in <b>bold</b> when its results are also displayed in the fingerprints<br>");
113	"<li>an icon showing whether the test fails or passes and whether it's reliable or not.<br>\n" +	161	stream.println("and starts with an '*' when the scenario has no results in the last baseline run.<br><br>");
114	"The legend for this icon is:\n" +	162	stream.println("Here are information displayed for each test (ie. in each cell):");
115	"<ul>\n" +	163	stream.println("<ul>");
116	"<li>Green (<img src=\""+Utils.OK_IMAGE+"\">): mark a <b>successful result</b>, which means this test has neither significant performance regression nor significant standard error</li>\n" +	164	stream.println("<li>an icon showing whether the test fails or passes and whether it's reliable or not.<br>");
117	"<li>Red (<img src=\""+Utils.FAIL_IMAGE+"\">): mark a <b>failing result</b>, which means this test shows a significant performance regression (more than 10%)</li>\n" +	165	stream.println("The legend for this icon is:");
118	"<li>Gray (<img src=\""+Utils.FAIL_IMAGE_EXPLAINED+"\">): mark a <b>failing result</b> (see above) with a comment explaining this degradation.</li>\n" +	166	stream.println("<ul>");
119	"<li>Yellow (<img src=\""+Utils.FAIL_IMAGE_WARN+"\"> or <img src=\""+Utils.OK_IMAGE_WARN+"\">): mark a <b>failing or successful result</b> with a significant standard error (more than "+Utils.STANDARD_ERROR_THRESHOLD_STRING+")</li>\n" +	167	stream.print("<li>Green (<img src=\"");
120	"<li>\"n/a\": mark a test for with <b>no</b> performance results</li>\n" +	168	stream.print(Utils.OK_IMAGE);
121	"</ul></li>\n" +	169	stream.print("\">): mark a <b>successful result</b>, which means this test has neither significant performance regression nor significant standard error</li>");
122	"<li>the value of the deviation from the baseline as a percentage (ie. formula is: <code>(build_test_time - baseline_test_time) / baseline_test_time</code>)</li>\n" +	170	stream.print("<li>Red (<img src=\"");
123	"<li>the value of the standard error of this deviation as a percentage (ie. formula is: <code>sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time</code>)<br>\n" +	171	stream.print(Utils.FAIL_IMAGE);
124	"Note that errors equal to 0 are not shown (tests which have only one iteration).</li>\n" +	172	stream.println("\">): mark a <b>failing result</b>, which means this test shows a significant performance regression (more than 10%)</li>");
125	"</ul>" +	173	stream.print("<li>Gray (<img src=\"");
126	"For failing tests, value of deviation with its standard error is added at the beginning of the error message you can see flying over the corresponding image.<br>\n" +	174	stream.print(Utils.FAIL_IMAGE_EXPLAINED);
127	"Follow the link on test box corresponding image for detailed results.<br>" +	175	stream.println("\">): mark a <b>failing result</b> (see above) with a comment explaining this degradation.</li>");
128	"<br>\n");	176	stream.print("<li>Yellow (<img src=\"");
		177	stream.print(Utils.FAIL_IMAGE_WARN);
		178	stream.print("\"> or <img src=\"");
		179	stream.print(Utils.OK_IMAGE_WARN);
		180	stream.print("\">): mark a <b>failing or successful result</b> with a significant standard error (more than ");
		181	stream.print(Utils.STANDARD_ERROR_THRESHOLD_STRING);
		182	stream.println(")</li>");
		183	stream.print("<li>Black (<img src=\"");
		184	stream.print(Utils.UNKNOWN_IMAGE);
		185	stream.print("\">): mark an <b>undefined result</b>, which means that deviation on this test is not a number (<code>NaN</code>) or is infinite (happens when the reference value is equals to 0!)</li>");
		186	stream.println("<li>\"n/a\": mark a test for with <b>no</b> performance results</li>");
		187	stream.println("</ul></li>");
		188	stream.println("<li>the value of the deviation from the baseline as a percentage (ie. formula is: <code>(build_test_time - baseline_test_time) / baseline_test_time</code>)</li>");
		189	stream.println("<li>the value of the standard error of this deviation as a percentage (ie. formula is: <code>sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time</code>)<br>");
		190	stream.println("When test only has one measure, the standard error cannot be computed and is replaced with a '<font color=\"#CCCC00\">[n/a]</font>'.</li>");
		191	stream.println("</ul>");
		192	stream.println("<u>Hints</u>:<ul>");
		193	stream.println("<li>fly over image of failing tests to see the complete error message</li>");
		194	stream.println("<li>to look at the complete and detailed test results, click on its image</li>");
		195	stream.println("</ul>");
		196	stream.println();
		197	stream.println("<table border=\"1\">");
		198	stream.println("<tr>");
		199	stream.print("<td><h4>All ");
		200	stream.print(scenarios.length);
		201	stream.println(" scenarios</h4></td>");
129		202
130	htmlTable=htmlTable.concat("<table border=\"1\"><tr><td><h4>All "+scenarios.length+" scenarios</h4></td>\n");
131	for (int i= 0; i < configNames.size(); i++){	203	for (int i= 0; i < configNames.size(); i++){
132	label=configNames.get(i).toString();	204	label = configNames.get(i).toString();
133	String columnTitle=label;	205	String columnTitle = label;
134	if (configMaps!=null) {	206	if (configMaps!=null) {
135	Utils.ConfigDescriptor configDescriptor= (Utils.ConfigDescriptor)configMaps.get(label);	207	Utils.ConfigDescriptor configDescriptor= (Utils.ConfigDescriptor)configMaps.get(label);
136	if (configDescriptor != null)	208	if (configDescriptor != null) {
137	columnTitle=configDescriptor.description;	209	int idx = configDescriptor.description.indexOf('(');
		210	if (idx < 0) {
		211	columnTitle = configDescriptor.description;
		212	} else {
		213	// first line
		214	StringTokenizer tokenizer = new StringTokenizer(configDescriptor.description.substring(0, idx).trim(), " ");
		215	StringBuffer buffer = new StringBuffer(tokenizer.nextToken());
		216	while (tokenizer.hasMoreTokens()) {
		217	buffer.append(" ");
		218	buffer.append(tokenizer.nextToken());
		219	}
		220	buffer.append(' ');
		221	// second line
		222	tokenizer = new StringTokenizer(configDescriptor.description.substring(idx).trim(), " ");
		223	buffer.append(tokenizer.nextToken());
		224	while (tokenizer.hasMoreTokens()) {
		225	buffer.append(" ");
		226	buffer.append(tokenizer.nextToken());
		227	}
		228	columnTitle = buffer.toString();
		229	}
		230	}
138	}	231	}
139	htmlTable=htmlTable.concat("<td><h5>"+columnTitle +"</h5></td>");	232	stream.print("<td><h5>");
		233	stream.print(columnTitle);
		234	stream.println("</h5>");
140	}	235	}
141
142	htmlTable=htmlTable.concat("</tr>\n");
143		236
144	// counter for js class Id's	237	// counter for js class Id's
145	int jsIdCount=0;	238	int jsIdCount=0;
Lines 147-207 Link Here
147		240
148	ScenarioStatus status=(ScenarioStatus)scenarioStatusList.get(j);	241	ScenarioStatus status=(ScenarioStatus)scenarioStatusList.get(j);
149		242
150	htmlTable=htmlTable.concat("<tr><td>"+status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1)+"</td>");	243	stream.println("<tr>");
		244	stream.print("<td>");
		245	if (status.fingerprint) stream.print("<b>");
		246	if (!status.hasBaseline) stream.print("*");
		247	// stream.print(status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1));
		248	stream.print(status.shortName);
		249	if (!status.hasBaseline) stream.print("</i>");
		250	if (status.fingerprint) stream.print("</b>");
		251	stream.println();
151	for (int i=0;i<configNames.size();i++){	252	for (int i=0;i<configNames.size();i++){
152	String message=null;
153	String configName=configNames.get(i).toString();	253	String configName=configNames.get(i).toString();
154	String aUrl=configName;	254	String aUrl=configName;
155	double[] resultStats = null;
156	if(status.statusMap.get(configName)!=null){
157	Object[] statusInfo = (Object[]) status.statusMap.get(configName);
158	message = (String) statusInfo[0];
159	resultStats = (double[]) statusInfo[1];
160	}
161
162	if (status.statusMap.containsKey(configName)){	255	if (status.statusMap.containsKey(configName)){
		256	String message = (String) status.statusMap.get(configName);
163	int confidence = ((Integer) status.configStatus.get(configName)).intValue();	257	int confidence = ((Integer) status.configStatus.get(configName)).intValue();
164	String image = Utils.getImage(confidence, resultStats, status.hasSlowDownExplanation);	258	String image = Utils.getImage(confidence, status.hasSlowDownExplanation);
165	StringBuffer html = new StringBuffer("\n<td><a ");	259	stream.print("<td><a ");
166	if ((confidence & Utils.DEV) == 0 \|\| message.length() == 0){	260	if ((confidence & Utils.DEV) == 0 \|\| (confidence & Utils.NAN) != 0 \|\| message.length() == 0){
167	// write deviation with error in table when test pass	261	// write deviation with error in table when test pass
168	html.append("href=\"");	262	stream.print("href=\"");
169	html.append(aUrl);	263	stream.print(aUrl);
170	html.append('/');	264	stream.print('/');
171	html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));	265	stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
172	html.append(".html\">\n<img hspace=\"10\" border=\"0\" src=\"");	266	stream.println(".html\">");
173	html.append(image);	267	stream.print("<img hspace=\"10\" border=\"0\" src=\"");
174	html.append("\"/></a>");	268	stream.print(image);
		269	stream.println("\"/></a>");
175	} else {	270	} else {
176	// create message with tooltip text including deviation with error plus failure message	271	// create message with tooltip text including deviation with error plus failure message
177	jsIdCount+=1;	272	jsIdCount+=1;
178	html.append("class=\"tooltipSource\" onMouseover=\"show_element('toolTip");	273	stream.print("class=\"tooltipSource\" onMouseover=\"show_element('toolTip");
179	html.append(jsIdCount);	274	stream.print(jsIdCount);
180	html.append("')\" onMouseout=\"hide_element('toolTip");	275	stream.print("')\" onMouseout=\"hide_element('toolTip");
181	html.append(jsIdCount);	276	stream.print(jsIdCount);
182	html.append("')\" \nhref=\"");	277	stream.print("')\" \nhref=\"");
183	html.append(aUrl);	278	stream.print(aUrl);
184	html.append('/');	279	stream.print('/');
185	html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));	280	stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
186	html.append(".html\">\n<img hspace=\"10\" border=\"0\" src=\"");	281	stream.println(".html\">");
187	html.append(image);	282	stream.print("<img hspace=\"10\" border=\"0\" src=\"");
188	html.append("\"/>\n<span class=\"hidden_tooltip\" id=\"toolTip");	283	stream.print(image);
189	html.append(jsIdCount);	284	stream.println("\"/>");
190	html.append("\">");	285	stream.print("<span class=\"hidden_tooltip\" id=\"toolTip");
191	html.append(message);	286	stream.print(jsIdCount);
192	html.append("</span></a>");	287	stream.print("\">");
		288	stream.print(message);
		289	stream.println("</span></a>");
193	}	290	}
194	html.append(Utils.failureMessage(resultStats, false));	291	String result = (String) status.resultsMap.get(configName);
195	html.append("</td>");	292	stream.println(result);
196	htmlTable=htmlTable.concat(html.toString());
197	}else{	293	}else{
198	htmlTable=htmlTable.concat("<td>n/a</td>");	294	stream.println("<td> n/a");
199	}	295	}
200	}	296	}
		297	stream.flush();
201	}	298	}
202		299	stream.println("</table>");
203	htmlTable=htmlTable.concat("</tr>\n");
204	}	300	}
205	return htmlTable;	301	}
206	}
207	}	302	}




import java.util.Hashtable;

import org.eclipse.swt.SWT;
import org.eclipse.swt.events.PaintEvent;
import org.eclipse.swt.events.PaintListener;
import org.eclipse.swt.graphics.GC;
import org.eclipse.swt.graphics.Image;
import org.eclipse.swt.graphics.ImageData;
import org.eclipse.swt.graphics.ImageLoader;
import org.eclipse.swt.graphics.Point;
import org.eclipse.swt.layout.FillLayout;
import org.eclipse.swt.widgets.Canvas;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.test.internal.performance.PerformanceTestPlugin;
import org.eclipse.test.internal.performance.data.Dim;
import org.eclipse.test.internal.performance.db.DB;

import org.eclipse.test.internal.performance.db.SummaryEntry;
import org.eclipse.test.internal.performance.db.TimeSeries;
import org.eclipse.test.internal.performance.db.Variations;
import org.eclipse.test.internal.performance.eval.StatisticsUtil;
import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
import org.eclipse.test.performance.Dimension;
import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;



        if (component==null){
        	entries= DB.querySummaries(variations,null);
        	this.component="";
        } else {
        else
        	entries=DB.querySummaries(variations,component+'%');
        }
       	run();
    }    
    
    /**
     * Creates the fingerprint gif, image map and scenario status table for the component.
     * @param entries - the result of a database query for summaries for a specified variation.
     */
    public void run() {
        new File(outputDirectory).mkdirs();
        String referenceName=referenceBuildId;
        String currentName=currentBuildId;

                
        if (entries != null) {
            for (int i= 0; i < entries.length; i++) {
                SummaryEntry summary = entries[i];
                if (summary.comment==null)
                	add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName);
                else{
                	setComment(summary.scenarioName, summary.comment);
                	add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName, summary.comment);
                }
            }
        }

               
        if (component=="")
        	outName= "FP_"+referenceName + '_' + currentBuildId+"."+configDescriptor.name;
        save(outputDirectory + '/' + outName);
        
        //show(bar);
     

    	scenarioComments.put(scenario,comment);
    }

    private void add(String name, Dim[] dims, String scenarioName) {
    	add (name,dims,scenarioName,null);
    }

    private void add(String name, Dim[] dims, String scenarioName, String comment) {
         String refData= "";
        Scenario scenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, referenceBuildId, currentBuildId, dims);
        String[] timeSeriesLabels= scenario.getTimeSeriesLabels();

            	double percent= 0.0;
            	boolean hasConfidentResult= true;
            	if (l > 1) {
//            		hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name);
            		hasConfidentResult= Utils.hasConfidentResult(timeSeries);
            		/*if (!rejectNullHypothesis) {
            			NumberFormat percentFormatter= NumberFormat.getPercentInstance();
            			String statisticsComment= "There is not enough evidence to reject the null hypothesis at the " + percentFormatter.format(percentile.inside()) + "level";

	         
    }

    private void save(String output) {

    	//if (bar.getFItems().size()==0)
    		//return;

    
    /*
     * Displays bar graph in window
     *
    private void show(final BarGraph bar) {
        Display display= new Display();
        

	public String getOutName() {
		return outName;
	}
    */

	/**
	 * @return - an html representation of the fingerprint.
	 */
	public String getImageMap() {
		String componentDescription = this.configDescriptor.description;
		String areas = this.bar.getAreas();
		if (areas == null)
			areas = "";
		String output = "";
		if (new File(this.outputDirectory, this.outName + ".gif").exists()) {
			output = "<h4>" + componentDescription + "</h4>";
			output = output.concat("<img src=\"" + this.outName + ".gif\" usemap=\"#" + this.outName + "\">" + "<map name=\"" + this.outName + "\">" + areas + "</map>\n");
		} else {
			output = output.concat("<br><br>There is no fingerprint for " + componentDescription + "<br><br>\n");
		}
		return output;
	}
}




import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Hashtable;

	 * 
	 * @param scenarios -
	 *            the array of Scenario objects for which to generate results.
	 * @param baseline -
	 *            the reference build ID
	 * @param current -
	 *            the current buildID

		this.outputDir=outputDir+"/"+configDescriptor.name;
		variabilityData=variabilityTable;

		try {
			printSummary();
			printDetails();
		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}

	private void printSummary() {




import org.eclipse.core.runtime.Platform;

import org.eclipse.test.internal.performance.db.Scenario;
import org.eclipse.test.internal.performance.db.SummaryEntry;
import org.eclipse.test.internal.performance.db.Variations;
import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;


	private Hashtable fingerPrints = new Hashtable();
	private Hashtable scenarioComments=new Hashtable();
	private Hashtable rawDataTables=new Hashtable();
	private boolean local = false;
	
	public Object run(Object args) throws Exception {
		parse(args);
		
		try {
			if (this.local) {
				int length = configNames.length;
				for (int i=0; i<length; i++) {
					generate((ConfigDescriptor)configDescriptors.get(configNames[i]));
				}
			} else {
				Enumeration configIds=configDescriptors.keys();
				while (configIds.hasMoreElements()){
					generate((ConfigDescriptor)configDescriptors.get(configIds.nextElement()));
				}
			}

			Utils.printVariabilityTable(rawDataTables,output+"/cvsummary.html",configDescriptors);
		
			Enumeration components = fingerPrints.keys();
			Bundle bundle= UiPlugin.getDefault().getBundle();
			URL images=bundle.getEntry("images");
			URL scripts=bundle.getEntry("scripts");

			if (images!=null) {
				images= Platform.resolve(images);
				Utils.copyImages(new File(images.getPath()), new File(output));
			}
			if (scripts!=null){
				scripts= Platform.resolve(scripts);
				Utils.copyScripts(new File(scripts.getPath()), new File(output));
			}
		
			// print fingerprint/scenario status pages
			while (components.hasMoreElements()) {
			String component = components.nextElement().toString();
			try {		
				File outputFile = new File(output, component + ".php");
				outputFile.getParentFile().mkdirs();
				PrintStream os = new PrintStream(new FileOutputStream(outputFile));

				os.println(Utils.HTML_DEFAULT_CSS);
				os.println("<body>");
				Hashtable fps = (Hashtable) fingerPrints.get(component);
				Enumeration configs = fps.keys();
				
				int baselineUnderScoreIndex=baseline.indexOf("_");
				int currentUnderScoreIndex=currentBuildId.indexOf("_");

				String baselineName=(baselineUnderScoreIndex!=-1)?baseline.substring(0, baseline.indexOf("_")):baseline;
				String currentName=(currentUnderScoreIndex!=-1)?currentBuildId.substring(0, currentBuildId.indexOf("_")):currentBuildId;
				boolean isGlobal = component.equals("global");
				StringBuffer title = new StringBuffer("<h3>Performance of ");
				if (!isGlobal) {
					title.append(component);
					title.append(": ");
				}
				title.append(currentName);
				title.append(" relative to ");
				title.append(baselineName);
				title.append( "</h3>");
				os.println(title.toString());
				
				//print the html representation of fingerprint for each config 
				Enumeration configs = fps.keys();
				SummaryEntry[] fpSummaries = null;
				while (configs.hasMoreElements()) {
					String config = configs.nextElement().toString();
					FingerPrint fingerPrint = (FingerPrint) fps.get(config);
					os.println(fingerPrint.getImageMap());
					if (fpSummaries == null) {
						fpSummaries = fingerPrint.entries;
					}
				}
				if (isGlobal) {
					if (this.local) {
						os.println("<table border=0 cellpadding=2 cellspacing=5 width=\"100%\">");
						os.println("<tbody><tr> <td colspan=3 align=\"left\" bgcolor=\"#0080c0\" valign=\"top\"><b><font color=\"#ffffff\" face=\"Arial,Helvetica\">");
						os.println("Detailed performance data grouped by scenario prefix</font></b></td></tr></tbody></table>");
						os.println("<a href=\"org.eclipse.ant.php?\">org.eclipse.ant*</a><br>");
						os.println("<a href=\"org.eclipse.compare.php?\">org.eclipse.compare*</a><br>");
						os.println("<a href=\"org.eclipse.core.php?\">org.eclipse.core*</a><br>");
						os.println("<a href=\"org.eclipse.help.php?\">org.eclipse.help*</a><br>");
						os.println("<a href=\"org.eclipse.jdt.core.php?\">org.eclipse.jdt.core*</a><br>");
						os.println("<a href=\"org.eclipse.jdt.debug.php?\">org.eclipse.jdt.debug*</a><br>");
						os.println("<a href=\"org.eclipse.jdt.text.php?\">org.eclipse.jdt.text*</a><br>");
						os.println("<a href=\"org.eclipse.jdt.ui.php?\">org.eclipse.jdt.ui*</a><br>");
						os.println("<a href=\"org.eclipse.jface.php?\">org.eclipse.jface*</a><br>");
						os.println("<a href=\"org.eclipse.osgi.php?\">org.eclipse.osgi*</a><br>");
						os.println("<a href=\"org.eclipse.pde.ui.php?\">org.eclipse.pde.ui*</a><br>");
						os.println("<a href=\"org.eclipse.swt.php?\">org.eclipse.swt*</a><br>");
						os.println("<a href=\"org.eclipse.team.php?\">org.eclipse.team*</a><br>");
						os.println("<a href=\"org.eclipse.ua.php?\">org.eclipse.ua*</a><br>");
						os.println("<a href=\"org.eclipse.ui.php?\">org.eclipse.ui*</a><br><p><br><br>");
					}
				} else if (component.length() > 0) {
					// print the component scenario status table beneath the fingerprint
					variations.put("config", "%");
					boolean filter = this.local && this.scenarioFilter != null; // use scenario filter to minimize DB requests while testing...
					ScenarioStatusTable sst = filter
						? new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, fpSummaries, baseline)
						: new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, fpSummaries, baseline);
					sst.print(os, filter);
				}

				os.println(Utils.HTML_CLOSE);
				os.close();
			} catch (Exception e) {
				// Need to print any unexpected exception otherwise the failure will be completely silent...
				e.printStackTrace();
			}
		} catch (Exception e) {
			// Need to print any unexpected exception otherwise the failure will be completely silent...
			e.printStackTrace();
		}

		return null;

					System.out.println("Missing value for -config.properties parameter");
					printUsage();
				}
				configDescriptors = Utils.getConfigDescriptors(configProperties);
						.getConfigDescriptors(configProperties);
				i++;
				continue;
			}

				i++;
				continue;
			}
			if (arg.equals("-local")) {
				this.local  = true;
				i++;
				continue;
			}

			i++;
		}




/*******************************************************************************
 * Copyright (c) 2000, 2007 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBM Corporation - initial API and implementation
 *******************************************************************************/
package org.eclipse.test.performance.ui;

import java.text.MessageFormat;

import org.eclipse.osgi.util.NLS;

public class Messages extends NLS {

	private static final String BUNDLE_NAME = "org.eclipse.test.performance.ui.messages";//$NON-NLS-1$

	private Messages() {
		// Do not instantiate
	}

	public static String standardError;

	static {
		NLS.initializeMessages(BUNDLE_NAME, Messages.class);
	}
	
	/**
	 * Bind the given message's substitution locations with the given string values.
	 * 
	 * @param message the message to be manipulated
	 * @return the manipulated String
	 */
	public static String bind(String message) {
		return bind(message, null);
	}
	
	/**
	 * Bind the given message's substitution locations with the given string values.
	 * 
	 * @param message the message to be manipulated
	 * @param binding the object to be inserted into the message
	 * @return the manipulated String
	 */
	public static String bind(String message, Object binding) {
		return bind(message, new Object[] {binding});
	}

	/**
	 * Bind the given message's substitution locations with the given string values.
	 * 
	 * @param message the message to be manipulated
	 * @param binding1 An object to be inserted into the message
	 * @param binding2 A second object to be inserted into the message
	 * @return the manipulated String
	 */
	public static String bind(String message, Object binding1, Object binding2) {
		return bind(message, new Object[] {binding1, binding2});
	}

	/**
	 * Bind the given message's substitution locations with the given string values.
	 * 
	 * @param message the message to be manipulated
	 * @param bindings An array of objects to be inserted into the message
	 * @return the manipulated String
	 */
	public static String bind(String message, Object[] bindings) {
		return MessageFormat.format(message, bindings);
	}

}




###############################################################################
# Copyright (c) 2000, 2007 IBM Corporation and others.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
#     IBM Corporation - initial API and implementation
###############################################################################

### Performance Tests Messages.

standardError = Standard error on this test is {0} (should be less than {1} to become reliable!)

Return to bug 185580

Lines 20-32 Link Here

(-)src/org/eclipse/test/performance/ui/Utils.java (-95 / +167 lines)
20	import java.io.InputStream;	20	import java.io.InputStream;
21	import java.io.OutputStream;	21	import java.io.OutputStream;
22	import java.io.PrintWriter;	22	import java.io.PrintWriter;
		23	import java.text.DecimalFormat;
23	import java.text.NumberFormat;	24	import java.text.NumberFormat;
24	import java.util.ArrayList;	25	import java.util.ArrayList;
25	import java.util.Arrays;	26	import java.util.Arrays;
26	import java.util.Calendar;	27	import java.util.Calendar;
27	import java.util.HashMap;	28	import java.util.HashMap;
28	import java.util.Hashtable;	29	import java.util.Hashtable;
29	import java.util.Map;
30	import java.util.StringTokenizer;	30	import java.util.StringTokenizer;
31		31
32	import junit.framework.AssertionFailedError;	32	import junit.framework.AssertionFailedError;
Lines 43-52 Link Here
43	import org.eclipse.test.internal.performance.data.Dim;	43	import org.eclipse.test.internal.performance.data.Dim;
44	import org.eclipse.test.internal.performance.db.DB;	44	import org.eclipse.test.internal.performance.db.DB;
45	import org.eclipse.test.internal.performance.db.Scenario;	45	import org.eclipse.test.internal.performance.db.Scenario;
		46	import org.eclipse.test.internal.performance.db.SummaryEntry;
46	import org.eclipse.test.internal.performance.db.TimeSeries;	47	import org.eclipse.test.internal.performance.db.TimeSeries;
47	import org.eclipse.test.internal.performance.db.Variations;	48	import org.eclipse.test.internal.performance.db.Variations;
48	import org.eclipse.test.internal.performance.eval.StatisticsUtil;
49	import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
50	import org.eclipse.test.performance.Dimension;	49	import org.eclipse.test.performance.Dimension;
51		50
52		51
Lines 57-78 Link Here
57	static {	56	static {
58	PERCENT_FORMAT.setMaximumFractionDigits(1);	57	PERCENT_FORMAT.setMaximumFractionDigits(1);
59	}	58	}
60	static final NumberFormat DOUBLE_FORMAT = NumberFormat.getNumberInstance();	59	static final DecimalFormat DEVIATION_FORMAT = (DecimalFormat) NumberFormat.getPercentInstance();
61	static {	60	static {
62	DOUBLE_FORMAT.setMaximumIntegerDigits(2);	61	DEVIATION_FORMAT.setMaximumFractionDigits(1);
63	DOUBLE_FORMAT.setMaximumFractionDigits(1);	62	DEVIATION_FORMAT.setMinimumFractionDigits(1);
		63	DEVIATION_FORMAT.setPositivePrefix("+");
		64	DEVIATION_FORMAT.setNegativePrefix("- ");
		65	}
		66	static final DecimalFormat STDERR_FORMAT = (DecimalFormat) NumberFormat.getNumberInstance();
		67	static {
		68	STDERR_FORMAT.setMaximumFractionDigits(1);
		69	STDERR_FORMAT.setMinimumFractionDigits(1);
		70	STDERR_FORMAT.setMultiplier(100);
64	}	71	}
65	public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);	72	public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);
66	public final static String STANDARD_ERROR_MESSAGE="Standard error on this test is higher than "+STANDARD_ERROR_THRESHOLD_STRING;	73	public final static String UNKNOWN_IMAGE="Unknown.gif";
67	public final static String OK_IMAGE="OK.gif";	74	public final static String OK_IMAGE="OK.gif";
68	public final static String OK_IMAGE_WARN="OK_caution.gif";	75	public final static String OK_IMAGE_WARN="OK_caution.gif";
69	public final static String FAIL_IMAGE="FAIL.gif";	76	public final static String FAIL_IMAGE="FAIL.gif";
70	public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";	77	public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";
71	public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";	78	public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";
72	public final static int OK = 0;	79	public final static int OK = 0;
73	public final static int SIGN = 0x1;	80	public final static int NAN = 0x1;
74	public final static int ERR = 0x2;	81	public final static int ERR = 0x2;
75	// public final static int TTEST = 0x2;
76	public final static int DEV = 0x4;	82	public final static int DEV = 0x4;
77		83
78	/**	84	/**
Lines 207-213 Link Here
207		213
208	while (tokenizer.hasMoreTokens()) {	214	while (tokenizer.hasMoreTokens()) {
209	String labelDescriptor = tokenizer.nextToken();	215	String labelDescriptor = tokenizer.nextToken();
210	String[] elements = labelDescriptor.split(",");	216	String[] elements = labelDescriptor.trim().split(",");
211	ConfigDescriptor descriptor = new ConfigDescriptor(elements[0], elements[1]);	217	ConfigDescriptor descriptor = new ConfigDescriptor(elements[0], elements[1]);
212	configMap.put(elements[0], descriptor);	218	configMap.put(elements[0], descriptor);
213	}	219	}
Lines 268-278 Link Here
268	return componentNames;	274	return componentNames;
269	}	275	}
270		276
271	/**	277	/*
272	* @param fp -	278	* @param fp -
273	* a FingerPrint object	279	* a FingerPrint object
274	* @return - an html representation of the fingerprint.	280	* @return - an html representation of the fingerprint.
275	*/	281	*
276	public static String getImageMap(FingerPrint fp) {	282	public static String getImageMap(FingerPrint fp) {
277	String componentDescription = fp.configDescriptor.description;	283	String componentDescription = fp.configDescriptor.description;
278	String areas = fp.bar.getAreas();	284	String areas = fp.bar.getAreas();
Lines 287-292 Link Here
287	}	293	}
288	return output;	294	return output;
289	}	295	}
		296	*/
290		297
291	/**	298	/**
292	* Utility method to copy a file.	299	* Utility method to copy a file.
Lines 314-324 Link Here
314	}	321	}
315	}	322	}
316	public static void copyImages(File images, File output) {	323	public static void copyImages(File images, File output) {
317	copyFile(new File(images, Utils.FAIL_IMAGE), new File(output, Utils.FAIL_IMAGE));	324	copyFile(new File(images, FAIL_IMAGE), new File(output, FAIL_IMAGE));
318	copyFile(new File(images, Utils.FAIL_IMAGE_EXPLAINED), new File(output, Utils.FAIL_IMAGE_EXPLAINED));	325	copyFile(new File(images, FAIL_IMAGE_EXPLAINED), new File(output, FAIL_IMAGE_EXPLAINED));
319	copyFile(new File(images, Utils.FAIL_IMAGE_WARN), new File(output, Utils.FAIL_IMAGE_WARN));	326	copyFile(new File(images, FAIL_IMAGE_WARN), new File(output, FAIL_IMAGE_WARN));
320	copyFile(new File(images, Utils.OK_IMAGE), new File(output, Utils.OK_IMAGE));	327	copyFile(new File(images, OK_IMAGE), new File(output, OK_IMAGE));
321	copyFile(new File(images, Utils.OK_IMAGE_WARN), new File(output, Utils.OK_IMAGE_WARN));	328	copyFile(new File(images, OK_IMAGE_WARN), new File(output, OK_IMAGE_WARN));
		329	copyFile(new File(images, UNKNOWN_IMAGE), new File(output, UNKNOWN_IMAGE));
322	}	330	}
323	public static void copyScripts(File scripts, File output) {	331	public static void copyScripts(File scripts, File output) {
324	copyFile(new File(scripts, "ToolTip.css"), new File(output, "ToolTip.css"));	332	copyFile(new File(scripts, "ToolTip.css"), new File(output, "ToolTip.css"));
Lines 433-439 Link Here
433	p.paint(image);	441	p.paint(image);
434		442
435	/* Downscale to 8 bit depth palette to save to gif */	443	/* Downscale to 8 bit depth palette to save to gif */
436	ImageData data = Utils.downSample(image);	444	ImageData data = downSample(image);
437	ImageLoader il = new ImageLoader();	445	ImageLoader il = new ImageLoader();
438	il.data = new ImageData[] { data };	446	il.data = new ImageData[] { data };
439	OutputStream out = null;	447	OutputStream out = null;
Lines 599-605 Link Here
599	* @return date/time in format YYYYMMDDHHMM, ie. 200504060010	607	* @return date/time in format YYYYMMDDHHMM, ie. 200504060010
600	*/	608	*/
601	public static long getDateFromBuildID(String buildId) {	609	public static long getDateFromBuildID(String buildId) {
602	return Utils.getDateFromBuildID(buildId, false);	610	return getDateFromBuildID(buildId, false);
603	}	611	}
604		612
605	public static long getDateFromBuildID(String buildId, boolean matchLast) {	613	public static long getDateFromBuildID(String buildId, boolean matchLast) {
Lines 656-662 Link Here
656	PrintWriter out=null;	664	PrintWriter out=null;
657	try {	665	try {
658	out = new PrintWriter(new FileWriter(new File(outputFile)));	666	out = new PrintWriter(new FileWriter(new File(outputFile)));
659	out.println(Utils.HTML_OPEN + "</head><body>\n");	667	out.println(HTML_OPEN + "</head><body>\n");
660	out.println("<h3>Summary of Elapsed Process Variation Coefficients</h3>\n"+	668	out.println("<h3>Summary of Elapsed Process Variation Coefficients</h3>\n"+
661	"<p> This table provides a bird's eye view of variability in elapsed process times\n"+	669	"<p> This table provides a bird's eye view of variability in elapsed process times\n"+
662	"for baseline and current build stream performance scenarios." +	670	"for baseline and current build stream performance scenarios." +
Lines 740-830 Link Here
740	out.close();	748	out.close();
741	}	749	}
742	}	750	}
743		751
744	public static double[] resultStats(Variations variations, String scenarioName, String baseline, String config) {	752	public static double[] resultsStatistics(TimeSeries timeSeries) {
745	String OS = "config";	753	try {
746		754	double valueRef = timeSeries.getValue(0), value = timeSeries.getValue(1);
747	Variations tmpVariations=(Variations)variations.clone();	755	long countRef = timeSeries.getCount(0), count = timeSeries.getCount(1);
748	tmpVariations.put(OS,config);	756	double stddevRef = timeSeries.getStddev(0), stddev = timeSeries.getStddev(1);
749	Scenario[] currentScenarios = DB.queryScenarios(tmpVariations, scenarioName,OS, null);	757	double stderr = (countRef == 1 \|\| count == 1)
750	Variations referenceVariations = (Variations) variations.clone();	758	? Double.NaN
751	referenceVariations.put(PerformanceTestPlugin.BUILD, baseline);	759	: (Double.isNaN(stddevRef)
752	referenceVariations.put(OS, config);	760	? Math.sqrt((stddev * stddev / count)) / valueRef
753	Scenario[] refScenarios = DB.queryScenarios(referenceVariations,	761	: Math.sqrt((stddevRef * stddevRef / countRef) + (stddev * stddev / count)) / valueRef);
754	scenarioName, OS, null);	762	return new double[] {
755		763	(value - valueRef) / valueRef,
756	Map referenceScenariosMap = new HashMap();	764	stderr,
757	Map currentScenariosMap = new HashMap();	765	};
758	for (int i = 0; i < refScenarios.length; i++) {	766	}
759	Scenario scenario = refScenarios[i];	767	catch (ArrayIndexOutOfBoundsException aioobe) {
760	String name = scenario.getScenarioName();	768	return null;
761	referenceScenariosMap.put(name, scenario);	769	}
762	}
763
764	for (int i = 0; i < currentScenarios.length; i++) {
765	Scenario scenario = currentScenarios[i];
766	String name = scenario.getScenarioName();
767	currentScenariosMap.put(name, scenario);
768	}
769	Percentile percentile = StatisticsUtil.T90;
770	Scenario scenario = (Scenario) currentScenariosMap.get(scenarioName);
771
772	Scenario reference = (Scenario) referenceScenariosMap.get(scenarioName);
773	if (reference != null) {
774	// XXX have to find out the relevant dimension
775	Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;
776	TimeSeries currentSeries = scenario.getTimeSeries(significanceDimension);
777	TimeSeries baselineSeries = reference.getTimeSeries(significanceDimension);
778	if (currentSeries.getLength() > 0 && baselineSeries.getLength() > 0) {
779	return StatisticsUtil.statisticsForTimeSeries(baselineSeries, 0, currentSeries, 0, percentile);
780	}
781	}
782	return null;
783	}	770	}
784		771
785	public static boolean hasConfidentResult(Variations variations, String scenarioName, String baseline, String config) {	772	public static boolean hasConfidentResult(TimeSeries timeSeries) {
786	double[] resultStats = resultStats(variations, scenarioName, baseline, config);	773	double[] resultStats = resultsStatistics(timeSeries);
787	return (confidenceLevel(resultStats) & ERR) == 0;	774	return (confidenceLevel(resultStats) & ERR) == 0;
788	}	775	}
789	public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) {	776	public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) {
790	return failureMessage(resultStats(variations, scenarioName, baseline, config), true);	777	String current = (String) variations.get(PerformanceTestPlugin.BUILD);
		778	Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;
		779	Scenario newScenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, baseline, current, new Dim[] { significanceDimension });
		780	TimeSeries timeSeries = newScenario.getTimeSeries(significanceDimension);
		781	double[] results = resultsStatistics(timeSeries);
		782	return failureMessage(results, true);
791	}	783	}
792	public static String failureMessage(double[] resultStats, boolean full) {	784	public static String failureMessage(double[] resultStats, boolean full) {
793	StringBuffer buffer = new StringBuffer();	785	StringBuffer buffer = new StringBuffer();
794	int level = confidenceLevel(resultStats);	786	int level = confidenceLevel(resultStats);
795	boolean signal = (level & SIGN) != 0;	787	// boolean isWarn = (level & WARN) != 0;
796	boolean isErr = (level & ERR) != 0;	788	boolean isErr = (level & ERR) != 0;
797	if (full & isErr) {	789	if (full) {
798	buffer.append("* WARNING * ");	790	if (isErr) {
799	buffer.append(STANDARD_ERROR_MESSAGE);	791	buffer.append("* WARNING * ");
		792	buffer.append(Messages.bind(Messages.standardError, PERCENT_FORMAT.format(resultStats[1]), STANDARD_ERROR_THRESHOLD_STRING));
		793	}
		794	return buffer.toString();
800	}	795	}
801	if (!full) buffer.append("<font color=\"#0000FF\" size=\"1\"> ");
802	if (resultStats != null) {	796	if (resultStats != null) {
803	double deviation = resultStats[3]==0 ? 0 : -resultStats[3];	797	double deviation = resultStats[0];
804	if (deviation > 0) {	798	buffer.append("<font color=\"#0000FF\" size=\"1\">");
805	buffer.append('+');	799	if (Double.isNaN(deviation) \|\| Double.isInfinite(deviation)) {
806	}	800	buffer.append(" [n/a]");
807	buffer.append(PERCENT_FORMAT.format(deviation));	801	} else {
808	if (signal) {	802	double stderr = resultStats[1];
809	buffer.append(" [±");	803	deviation = Math.abs(deviation)<0.001 ? 0 : -deviation;
810	buffer.append(DOUBLE_FORMAT.format(resultStats[2]*100));	804	if (Double.isNaN(stderr) \|\| Double.isInfinite(stderr)) {
811	buffer.append(']');	805	buffer.append(DEVIATION_FORMAT.format(deviation));
		806	buffer.append("</font><font color=\"#DDDD00\" size=\"1\"> ");
		807	buffer.append(" [n/a]");
		808	} else {
		809	buffer.append(DEVIATION_FORMAT.format(deviation));
		810	buffer.append(" [±");
		811	buffer.append(STDERR_FORMAT.format(Math.abs(stderr)));
		812	buffer.append(']');
		813	}
812	}	814	}
		815	buffer.append("</font>");
813	}	816	}
814	if (!full) buffer.append("</font>");
815	return buffer.toString();	817	return buffer.toString();
816	}	818	}
817	public static int confidenceLevel(double[] resultStats) {	819	public static int confidenceLevel(double[] resultStats) {
818	int level = OK;	820	int level = OK;
819	if (resultStats != null){	821	if (resultStats != null){
820	// if (resultStats[1] >= 0 && resultStats[0] >= resultStats[1]) { // invalid t-test	822	if (Double.isNaN(resultStats[0]) \|\| Double.isInfinite(resultStats[0])) {
821	// level \|= TTEST;	823	level = NAN;
822	// }	824	} else {
823	if (resultStats[2] > 0) { // signal standard error higher than 0% (only one iteration)	825	// if (resultStats[1] >= (STANDARD_ERROR_THRESHOLD/2)) { // warns standard error higher than the half of authorized threshold
824	level \|= SIGN;	826	// level \|= WARN;
825	}	827	// }
826	if (resultStats[2] >= Utils.STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold	828	if (resultStats[1] >= STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold
827	level \|= ERR;	829	level = ERR;
		830	}
828	}	831	}
829	}	832	}
830	return level;	833	return level;
Lines 837-851 Link Here
837	String previous = "";	840	String previous = "";
838	while (tokenizer.hasMoreTokens()) {	841	while (tokenizer.hasMoreTokens()) {
839	String token = tokenizer.nextToken();	842	String token = tokenizer.nextToken();
840	if (token.equals("%")) {	843	if (!token.equals("%") && !token.equals("_")) {
841	start += previous.length();
842	} else if (token.equals("_")) {
843	start++;
844	} else {
845	if (previous.equals("%")) {	844	if (previous.equals("%")) {
846	if (name.substring(start).indexOf(token) < 0) return false;	845	int idx = name.substring(start).indexOf(token);
		846	if (idx < 0) return false;
		847	start += idx;
847	} else if (previous.equals("_")) {	848	} else if (previous.equals("_")) {
848	if (!name.substring(start).startsWith(token)) return false;	849	if (!name.substring(++start).startsWith(token)) return false;
849	}	850	}
850	start += token.length();	851	start += token.length();
851	}	852	}
Lines 861-867 Link Here
861	return name.equals(pattern);	862	return name.equals(pattern);
862	}	863	}
863		864
864	public static String getImage(int confidence, double[] resultStats, boolean hasExplanation) {	865	public static String getImage(int confidence, boolean hasExplanation) {
865	boolean scenarioFailed = (confidence & DEV) != 0;	866	boolean scenarioFailed = (confidence & DEV) != 0;
866	String image = null;	867	String image = null;
867		868
Lines 873-878 Link Here
873	} else {	874	} else {
874	image = FAIL_IMAGE;	875	image = FAIL_IMAGE;
875	}	876	}
		877	} else if ((confidence & NAN) != 0) {
		878	image = UNKNOWN_IMAGE;
876	} else if ((confidence & ERR) != 0) {	879	} else if ((confidence & ERR) != 0) {
877	image = OK_IMAGE_WARN;	880	image = OK_IMAGE_WARN;
878	} else {	881	} else {
Lines 880-883 Link Here
880	}	883	}
881	return image;	884	return image;
882	}	885	}
883	}	886
		887	public static boolean hasSummary(SummaryEntry[] summaries, String scenarioName) {
		888	int length = summaries == null ? 0 : summaries.length;
		889	for (int i=0; i<length; i++) {
		890	SummaryEntry summary = summaries[i];
		891	if (summary.scenarioName.equals(scenarioName)) {
		892	return true;
		893	}
		894	}
		895	return false;
		896	}
		897
		898	public static String getScenarioShortName(String scenarioName, int max) {
		899
		900	// Remove class name qualification
		901	int testSeparator = scenarioName.indexOf('#');
		902	boolean hasClassName = testSeparator >= 0;
		903	if (!hasClassName) {
		904	testSeparator = scenarioName.lastIndexOf('.');
		905	if (testSeparator <= 0) {
		906	if (max > 0 && scenarioName.length() > max) {
		907	return "*"+scenarioName.substring(0, max);
		908	}
		909	return scenarioName;
		910	}
		911	}
		912	int classSeparator = scenarioName.substring(0, testSeparator).lastIndexOf('.');
		913	if (classSeparator < 0) {
		914	if (max > 0 && scenarioName.length() > max) {
		915	return "*"+scenarioName.substring(0, max);
		916	}
		917	return scenarioName;
		918	}
		919	int length = scenarioName.length();
		920	String shortName = scenarioName.substring(classSeparator+1, length);
		921	if (!hasClassName && shortName.startsWith("test.")) { // specific case for swt...
		922	shortName = shortName.substring(5);
		923	}
		924
		925	// Remove qualification from test name
		926	StringTokenizer tokenizer = new StringTokenizer(shortName, " :,", true);
		927	StringBuffer buffer = new StringBuffer(tokenizer.nextToken());
		928	while (tokenizer.hasMoreTokens()) {
		929	String token = tokenizer.nextToken();
		930	char fc = token.charAt(0);
		931	while (fc == ' ' \|\| fc == ',' \|\| fc == ':') {
		932	buffer.append(token); // add the separator
		933	token = tokenizer.nextToken();
		934	fc = token.charAt(0);
		935	}
		936	int last = token .lastIndexOf('.');
		937	if (last >= 3) {
		938	int first = token .indexOf('.');
		939	if (first == last) {
		940	buffer.append(token);
		941	} else {
		942	// buffer.append(token.substring(0, first));
		943	// buffer.append("...");
		944	buffer.append(token.substring(last+1));
		945	}
		946	} else {
		947	buffer.append(token);
		948	}
		949	}
		950	if (max > 0 && buffer.length() > max) {
951	return "*"+buffer.substring(0, max);
952	}
953	return buffer.toString();
954	}
955	}

Lines 19-35 Link Here

(-)src/org/eclipse/test/performance/ui/FingerPrint.java (-26 / +36 lines)
19	import java.util.Hashtable;	19	import java.util.Hashtable;
20		20
21	import org.eclipse.swt.SWT;	21	import org.eclipse.swt.SWT;
22	import org.eclipse.swt.events.PaintEvent;
23	import org.eclipse.swt.events.PaintListener;
24	import org.eclipse.swt.graphics.GC;	22	import org.eclipse.swt.graphics.GC;
25	import org.eclipse.swt.graphics.Image;	23	import org.eclipse.swt.graphics.Image;
26	import org.eclipse.swt.graphics.ImageData;	24	import org.eclipse.swt.graphics.ImageData;
27	import org.eclipse.swt.graphics.ImageLoader;	25	import org.eclipse.swt.graphics.ImageLoader;
28	import org.eclipse.swt.graphics.Point;
29	import org.eclipse.swt.layout.FillLayout;
30	import org.eclipse.swt.widgets.Canvas;
31	import org.eclipse.swt.widgets.Display;	26	import org.eclipse.swt.widgets.Display;
32	import org.eclipse.swt.widgets.Shell;
33	import org.eclipse.test.internal.performance.PerformanceTestPlugin;	27	import org.eclipse.test.internal.performance.PerformanceTestPlugin;
34	import org.eclipse.test.internal.performance.data.Dim;	28	import org.eclipse.test.internal.performance.data.Dim;
35	import org.eclipse.test.internal.performance.db.DB;	29	import org.eclipse.test.internal.performance.db.DB;
Lines 37-45 Link Here
37	import org.eclipse.test.internal.performance.db.SummaryEntry;	31	import org.eclipse.test.internal.performance.db.SummaryEntry;
38	import org.eclipse.test.internal.performance.db.TimeSeries;	32	import org.eclipse.test.internal.performance.db.TimeSeries;
39	import org.eclipse.test.internal.performance.db.Variations;	33	import org.eclipse.test.internal.performance.db.Variations;
40	import org.eclipse.test.internal.performance.eval.StatisticsUtil;
41	import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
42	import org.eclipse.test.performance.Dimension;
43	import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;	34	import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;
44		35
45		36
Lines 74-90 Link Here
74	if (component==null){	65	if (component==null){
75	entries= DB.querySummaries(variations,null);	66	entries= DB.querySummaries(variations,null);
76	this.component="";	67	this.component="";
77	}	68	} else {
78	else
79	entries=DB.querySummaries(variations,component+'%');	69	entries=DB.querySummaries(variations,component+'%');
80	run(entries);	70	}
		71	run();
81	}	72	}
82		73
83	/**	74	/**
84	* Creates the fingerprint gif, image map and scenario status table for the component.	75	* Creates the fingerprint gif, image map and scenario status table for the component.
85	* @param entries - the result of a database query for summaries for a specified variation.
86	*/	76	*/
87	public void run(Object[] entries) {	77	public void run() {
88	new File(outputDirectory).mkdirs();	78	new File(outputDirectory).mkdirs();
89	String referenceName=referenceBuildId;	79	String referenceName=referenceBuildId;
90	String currentName=currentBuildId;	80	String currentName=currentBuildId;
Lines 101-112 Link Here
101		91
102	if (entries != null) {	92	if (entries != null) {
103	for (int i= 0; i < entries.length; i++) {	93	for (int i= 0; i < entries.length; i++) {
104	SummaryEntry se= (SummaryEntry)entries[i];	94	SummaryEntry summary = entries[i];
105	if (se.comment==null)	95	if (summary.comment==null)
106	add(bar, se.shortName, new Dim[] { se.dimension }, se.scenarioName);	96	add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName);
107	else{	97	else{
108	setComment(se.scenarioName, se.comment);	98	setComment(summary.scenarioName, summary.comment);
109	add(bar, se.shortName, new Dim[] { se.dimension }, se.scenarioName,se.comment);	99	add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName, summary.comment);
110	}	100	}
111	}	101	}
112	}	102	}
Lines 115-121 Link Here
115		105
116	if (component=="")	106	if (component=="")
117	outName= "FP_"+referenceName + '_' + currentBuildId+"."+configDescriptor.name;	107	outName= "FP_"+referenceName + '_' + currentBuildId+"."+configDescriptor.name;
118	save(bar, outputDirectory + '/' + outName);	108	save(outputDirectory + '/' + outName);
119		109
120	//show(bar);	110	//show(bar);
121		111
Lines 127-137 Link Here
127	scenarioComments.put(scenario,comment);	117	scenarioComments.put(scenario,comment);
128	}	118	}
129		119
130	private void add(BarGraph bar, String name, Dim[] dims, String scenarioName) {	120	private void add(String name, Dim[] dims, String scenarioName) {
131	add (bar,name,dims,scenarioName,null);	121	add (name,dims,scenarioName,null);
132	}	122	}
133		123
134	private void add(BarGraph bar, String name, Dim[] dims, String scenarioName,String comment) {	124	private void add(String name, Dim[] dims, String scenarioName, String comment) {
135	String refData= "";	125	String refData= "";
136	Scenario scenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, referenceBuildId, currentBuildId, dims);	126	Scenario scenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, referenceBuildId, currentBuildId, dims);
137	String[] timeSeriesLabels= scenario.getTimeSeriesLabels();	127	String[] timeSeriesLabels= scenario.getTimeSeriesLabels();
Lines 153-159 Link Here
153	double percent= 0.0;	143	double percent= 0.0;
154	boolean hasConfidentResult= true;	144	boolean hasConfidentResult= true;
155	if (l > 1) {	145	if (l > 1) {
156	hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name);	146	// hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name);
		147	hasConfidentResult= Utils.hasConfidentResult(timeSeries);
157	/*if (!rejectNullHypothesis) {	148	/*if (!rejectNullHypothesis) {
158	NumberFormat percentFormatter= NumberFormat.getPercentInstance();	149	NumberFormat percentFormatter= NumberFormat.getPercentInstance();
159	String statisticsComment= "There is not enough evidence to reject the null hypothesis at the " + percentFormatter.format(percentile.inside()) + "level";	150	String statisticsComment= "There is not enough evidence to reject the null hypothesis at the " + percentFormatter.format(percentile.inside()) + "level";
Lines 178-184 Link Here
178		169
179	}	170	}
180		171
181	private void save(BarGraph bar, String output) {	172	private void save(String output) {
182		173
183	//if (bar.getFItems().size()==0)	174	//if (bar.getFItems().size()==0)
184	//return;	175	//return;
Lines 217-223 Link Here
217		208
218	/*	209	/*
219	* Displays bar graph in window	210	* Displays bar graph in window
220	*/	211	*
221	private void show(final BarGraph bar) {	212	private void show(final BarGraph bar) {
222	Display display= new Display();	213	Display display= new Display();
223		214
Lines 246-249 Link Here
246	public String getOutName() {	237	public String getOutName() {
247	return outName;	238	return outName;
248	}	239	}
		240	*/
		241
		242	/**
		243	* @return - an html representation of the fingerprint.
		244	*/
		245	public String getImageMap() {
		246	String componentDescription = this.configDescriptor.description;
		247	String areas = this.bar.getAreas();
		248	if (areas == null)
		249	areas = "";
		250	String output = "";
		251	if (new File(this.outputDirectory, this.outName + ".gif").exists()) {
		252	output = "<h4>" + componentDescription + "</h4>";
		253	output = output.concat("<img src=\"" + this.outName + ".gif\" usemap=\"#" + this.outName + "\">" + "<map name=\"" + this.outName + "\">" + areas + "</map>\n");
		254	} else {
		255	output = output.concat("<br><br>There is no fingerprint for " + componentDescription + "<br><br>\n");
		256	}
		257	return output;
		258	}
249	}	259	}

Lines 15-21 Link Here

(-)src/org/eclipse/test/performance/ui/ScenarioResults.java (-5 / +8 lines)
15	import java.io.FileNotFoundException;	15	import java.io.FileNotFoundException;
16	import java.io.FileOutputStream;	16	import java.io.FileOutputStream;
17	import java.io.PrintStream;	17	import java.io.PrintStream;
18	import java.text.NumberFormat;
19	import java.util.ArrayList;	18	import java.util.ArrayList;
20	import java.util.HashMap;	19	import java.util.HashMap;
21	import java.util.Hashtable;	20	import java.util.Hashtable;
Lines 48-54 Link Here
48	*	47	*
49	* @param scenarios -	48	* @param scenarios -
50	* the array of Scenario objects for which to generate results.	49	* the array of Scenario objects for which to generate results.
51	* @param reference -	50	* @param baseline -
52	* the reference build ID	51	* the reference build ID
53	* @param current -	52	* @param current -
54	* the current buildID	53	* the current buildID
Lines 76-84 Link Here
76	this.outputDir=outputDir+"/"+configDescriptor.name;	75	this.outputDir=outputDir+"/"+configDescriptor.name;
77	variabilityData=variabilityTable;	76	variabilityData=variabilityTable;
78		77
79	printSummary();	78	try {
80		79	printSummary();
81	printDetails();	80	printDetails();
		81	}
		82	catch (Exception e) {
		83	e.printStackTrace();
		84	}
82	}	85	}
83		86
84	private void printSummary() {	87	private void printSummary() {

Lines 25-30 Link Here

(-)src/org/eclipse/test/performance/ui/Main.java (-46 / +86 lines)
25	import org.eclipse.core.runtime.Platform;	25	import org.eclipse.core.runtime.Platform;
26		26
27	import org.eclipse.test.internal.performance.db.Scenario;	27	import org.eclipse.test.internal.performance.db.Scenario;
		28	import org.eclipse.test.internal.performance.db.SummaryEntry;
28	import org.eclipse.test.internal.performance.db.Variations;	29	import org.eclipse.test.internal.performance.db.Variations;
29	import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;	30	import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;
30		31
Lines 48-83 Link Here
48	private Hashtable fingerPrints = new Hashtable();	49	private Hashtable fingerPrints = new Hashtable();
49	private Hashtable scenarioComments=new Hashtable();	50	private Hashtable scenarioComments=new Hashtable();
50	private Hashtable rawDataTables=new Hashtable();	51	private Hashtable rawDataTables=new Hashtable();
		52	private boolean local = false;
51		53
52	public Object run(Object args) throws Exception {	54	public Object run(Object args) throws Exception {
53	parse(args);	55	parse(args);
54		56
55	Enumeration configIds=configDescriptors.keys();	57	try {
56		58	if (this.local) {
57	while (configIds.hasMoreElements()){	59	int length = configNames.length;
58	generate((ConfigDescriptor)configDescriptors.get(configIds.nextElement()));	60	for (int i=0; i<length; i++) {
59	}	61	generate((ConfigDescriptor)configDescriptors.get(configNames[i]));
		62	}
		63	} else {
		64	Enumeration configIds=configDescriptors.keys();
		65	while (configIds.hasMoreElements()){
		66	generate((ConfigDescriptor)configDescriptors.get(configIds.nextElement()));
		67	}
		68	}
60		69
61	Utils.printVariabilityTable(rawDataTables,output+"/cvsummary.html",configDescriptors);	70	Utils.printVariabilityTable(rawDataTables,output+"/cvsummary.html",configDescriptors);
62		71
63	Enumeration components = fingerPrints.keys();	72	Enumeration components = fingerPrints.keys();
64	Bundle bundle= UiPlugin.getDefault().getBundle();	73	Bundle bundle= UiPlugin.getDefault().getBundle();
65	URL images=bundle.getEntry("images");	74	URL images=bundle.getEntry("images");
66	URL scripts=bundle.getEntry("scripts");	75	URL scripts=bundle.getEntry("scripts");
67		76
68	if (images!=null) {	77	if (images!=null) {
69	images= Platform.resolve(images);	78	images= Platform.resolve(images);
70	Utils.copyImages(new File(images.getPath()), new File(output));	79	Utils.copyImages(new File(images.getPath()), new File(output));
71	}	80	}
72	if (scripts!=null){	81	if (scripts!=null){
73	scripts= Platform.resolve(scripts);	82	scripts= Platform.resolve(scripts);
74	Utils.copyScripts(new File(scripts.getPath()), new File(output));	83	Utils.copyScripts(new File(scripts.getPath()), new File(output));
75	}	84	}
76		85
77	// print fingerprint/scenario status pages	86	// print fingerprint/scenario status pages
78	while (components.hasMoreElements()) {	87	while (components.hasMoreElements()) {
79	String component = components.nextElement().toString();	88	String component = components.nextElement().toString();
80	try {
81	File outputFile = new File(output, component + ".php");	89	File outputFile = new File(output, component + ".php");
82	outputFile.getParentFile().mkdirs();	90	outputFile.getParentFile().mkdirs();
83	PrintStream os = new PrintStream(new FileOutputStream(outputFile));	91	PrintStream os = new PrintStream(new FileOutputStream(outputFile));
Lines 87-130 Link Here
87	os.println(Utils.HTML_DEFAULT_CSS);	95	os.println(Utils.HTML_DEFAULT_CSS);
88	os.println("<body>");	96	os.println("<body>");
89	Hashtable fps = (Hashtable) fingerPrints.get(component);	97	Hashtable fps = (Hashtable) fingerPrints.get(component);
90	Enumeration configs = fps.keys();
91		98
92	int baselineUnderScoreIndex=baseline.indexOf("_");	99	int baselineUnderScoreIndex=baseline.indexOf("_");
93	int currentUnderScoreIndex=currentBuildId.indexOf("_");	100	int currentUnderScoreIndex=currentBuildId.indexOf("_");
94		101
95	String baselineName=(baselineUnderScoreIndex!=-1)?baseline.substring(0, baseline.indexOf("_")):baseline;	102	String baselineName=(baselineUnderScoreIndex!=-1)?baseline.substring(0, baseline.indexOf("_")):baseline;
96	String currentName=(currentUnderScoreIndex!=-1)?currentBuildId.substring(0, currentBuildId.indexOf("_")):currentBuildId;	103	String currentName=(currentUnderScoreIndex!=-1)?currentBuildId.substring(0, currentBuildId.indexOf("_")):currentBuildId;
97	String title = "<h3>Performance of " + component + ": "	104	boolean isGlobal = component.equals("global");
98	+ currentName + " relative to "	105	StringBuffer title = new StringBuffer("<h3>Performance of ");
99	+ baselineName	106	if (!isGlobal) {
100	+ "</h3>";	107	title.append(component);
101	if (component.equals("global"))	108	title.append(": ");
102	title = "<h3>Performance of " + currentName	109	}
103	+ " relative to "	110	title.append(currentName);
104	+ baselineName	111	title.append(" relative to ");
105	+ "</h3>";	112	title.append(baselineName);
106	os.println(title);	113	title.append( "</h3>");
		114	os.println(title.toString());
107		115
108	//print the html representation of fingerprint for each config	116	//print the html representation of fingerprint for each config
		117	Enumeration configs = fps.keys();
		118	SummaryEntry[] fpSummaries = null;
109	while (configs.hasMoreElements()) {	119	while (configs.hasMoreElements()) {
110	String config = configs.nextElement().toString();	120	String config = configs.nextElement().toString();
111	FingerPrint fp = (FingerPrint) fps.get(config);	121	FingerPrint fingerPrint = (FingerPrint) fps.get(config);
112	os.println(Utils.getImageMap(fp));	122	os.println(fingerPrint.getImageMap());
113	}	123	if (fpSummaries == null) {
114	if (component != "") {	124	fpSummaries = fingerPrint.entries;
115	//print the component scenario status table beneath the fingerprint	125	}
		126	}
		127	if (isGlobal) {
		128	if (this.local) {
		129	os.println("<table border=0 cellpadding=2 cellspacing=5 width=\"100%\">");
		130	os.println("<tbody><tr> <td colspan=3 align=\"left\" bgcolor=\"#0080c0\" valign=\"top\"><b><font color=\"#ffffff\" face=\"Arial,Helvetica\">");
		131	os.println("Detailed performance data grouped by scenario prefix</font></b></td></tr></tbody></table>");
		132	os.println("<a href=\"org.eclipse.ant.php?\">org.eclipse.ant*</a><br>");
		133	os.println("<a href=\"org.eclipse.compare.php?\">org.eclipse.compare*</a><br>");
		134	os.println("<a href=\"org.eclipse.core.php?\">org.eclipse.core*</a><br>");
		135	os.println("<a href=\"org.eclipse.help.php?\">org.eclipse.help*</a><br>");
		136	os.println("<a href=\"org.eclipse.jdt.core.php?\">org.eclipse.jdt.core*</a><br>");
		137	os.println("<a href=\"org.eclipse.jdt.debug.php?\">org.eclipse.jdt.debug*</a><br>");
		138	os.println("<a href=\"org.eclipse.jdt.text.php?\">org.eclipse.jdt.text*</a><br>");
		139	os.println("<a href=\"org.eclipse.jdt.ui.php?\">org.eclipse.jdt.ui*</a><br>");
		140	os.println("<a href=\"org.eclipse.jface.php?\">org.eclipse.jface*</a><br>");
		141	os.println("<a href=\"org.eclipse.osgi.php?\">org.eclipse.osgi*</a><br>");
		142	os.println("<a href=\"org.eclipse.pde.ui.php?\">org.eclipse.pde.ui*</a><br>");
		143	os.println("<a href=\"org.eclipse.swt.php?\">org.eclipse.swt*</a><br>");
		144	os.println("<a href=\"org.eclipse.team.php?\">org.eclipse.team*</a><br>");
		145	os.println("<a href=\"org.eclipse.ua.php?\">org.eclipse.ua*</a><br>");
		146	os.println("<a href=\"org.eclipse.ui.php?\">org.eclipse.ui*</a><br><p><br><br>");
		147	}
		148	} else if (component.length() > 0) {
		149	// print the component scenario status table beneath the fingerprint
116	variations.put("config", "%");	150	variations.put("config", "%");
117	ScenarioStatusTable sst = new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, baseline);	151	boolean filter = this.local && this.scenarioFilter != null; // use scenario filter to minimize DB requests while testing...
118	// ScenarioStatusTable sst = new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, baseline);	152	ScenarioStatusTable sst = filter
119	os.println(sst.toString());	153	? new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, fpSummaries, baseline)
		154	: new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, fpSummaries, baseline);
		155	sst.print(os, filter);
120	}	156	}
121		157
122	os.println(Utils.HTML_CLOSE);	158	os.println(Utils.HTML_CLOSE);
123	os.close();	159	os.close();
124	} catch (Exception e) {
125	// Need to print any unexpected exception otherwise the failure will be completely silent...
126	e.printStackTrace();
127	}	160	}
		161	} catch (Exception e) {
		162	// Need to print any unexpected exception otherwise the failure will be completely silent...
		163	e.printStackTrace();
128	}	164	}
129		165
130	return null;	166	return null;
Lines 292-299 Link Here
292	System.out.println("Missing value for -config.properties parameter");	328	System.out.println("Missing value for -config.properties parameter");
293	printUsage();	329	printUsage();
294	}	330	}
295	configDescriptors = Utils	331	configDescriptors = Utils.getConfigDescriptors(configProperties);
296	.getConfigDescriptors(configProperties);
297	i++;	332	i++;
298	continue;	333	continue;
299	}	334	}
Lines 318-323 Link Here
318	i++;	353	i++;
319	continue;	354	continue;
320	}	355	}
		356	if (arg.equals("-local")) {
		357	this.local = true;
		358	i++;
		359	continue;
		360	}
321		361
322	i++;	362	i++;
323	}	363	}

Added Link Here

(-)src/org/eclipse/test/performance/ui/Messages.java (+75 lines)
1	/*******************************************************************************
2	* Copyright (c) 2000, 2007 IBM Corporation and others.
3	* All rights reserved. This program and the accompanying materials
4	* are made available under the terms of the Eclipse Public License v1.0
5	* which accompanies this distribution, and is available at
6	* http://www.eclipse.org/legal/epl-v10.html
7	*
8	* Contributors:
9	* IBM Corporation - initial API and implementation
10	*******************************************************************************/
11	package org.eclipse.test.performance.ui;
12
13	import java.text.MessageFormat;
14
15	import org.eclipse.osgi.util.NLS;
16
17	public class Messages extends NLS {
18
19	private static final String BUNDLE_NAME = "org.eclipse.test.performance.ui.messages";//$NON-NLS-1$
20
21	private Messages() {
22	// Do not instantiate
23	}
24
25	public static String standardError;
26
27	static {
28	NLS.initializeMessages(BUNDLE_NAME, Messages.class);
29	}
30
31	/**
32	* Bind the given message's substitution locations with the given string values.
33	*
34	* @param message the message to be manipulated
35	* @return the manipulated String
36	*/
37	public static String bind(String message) {
38	return bind(message, null);
39	}
40
41	/**
42	* Bind the given message's substitution locations with the given string values.
43	*
44	* @param message the message to be manipulated
45	* @param binding the object to be inserted into the message
46	* @return the manipulated String
47	*/
48	public static String bind(String message, Object binding) {
49	return bind(message, new Object[] {binding});
50	}
51
52	/**
53	* Bind the given message's substitution locations with the given string values.
54	*
55	* @param message the message to be manipulated
56	* @param binding1 An object to be inserted into the message
57	* @param binding2 A second object to be inserted into the message
58	* @return the manipulated String
59	*/
60	public static String bind(String message, Object binding1, Object binding2) {
61	return bind(message, new Object[] {binding1, binding2});
62	}
63
64	/**
65	* Bind the given message's substitution locations with the given string values.
66	*
67	* @param message the message to be manipulated
68	* @param bindings An array of objects to be inserted into the message
69	* @return the manipulated String
70	*/
71	public static String bind(String message, Object[] bindings) {
72	return MessageFormat.format(message, bindings);
73	}
74
75	}

Added Link Here

(-)src/org/eclipse/test/performance/ui/messages.properties (+14 lines)
1	###############################################################################
2	# Copyright (c) 2000, 2007 IBM Corporation and others.
3	# All rights reserved. This program and the accompanying materials
4	# are made available under the terms of the Eclipse Public License v1.0
5	# which accompanies this distribution, and is available at
6	# http://www.eclipse.org/legal/epl-v10.html
7	#
8	# Contributors:
9	# IBM Corporation - initial API and implementation
10	###############################################################################
11
12	### Performance Tests Messages.
13
14	standardError = Standard error on this test is {0} (should be less than {1} to become reliable!)