### Eclipse Workspace Patch 1.0 #P org.eclipse.test.performance.ui Index: src/org/eclipse/test/performance/ui/Utils.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/Utils.java,v retrieving revision 1.34 diff -u -r1.34 Utils.java --- src/org/eclipse/test/performance/ui/Utils.java 15 May 2007 19:23:10 -0000 1.34 +++ src/org/eclipse/test/performance/ui/Utils.java 31 May 2007 16:46:33 -0000 @@ -20,13 +20,13 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.PrintWriter; +import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.HashMap; import java.util.Hashtable; -import java.util.Map; import java.util.StringTokenizer; import junit.framework.AssertionFailedError; @@ -43,10 +43,9 @@ import org.eclipse.test.internal.performance.data.Dim; import org.eclipse.test.internal.performance.db.DB; import org.eclipse.test.internal.performance.db.Scenario; +import org.eclipse.test.internal.performance.db.SummaryEntry; import org.eclipse.test.internal.performance.db.TimeSeries; import org.eclipse.test.internal.performance.db.Variations; -import org.eclipse.test.internal.performance.eval.StatisticsUtil; -import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile; import org.eclipse.test.performance.Dimension; @@ -57,22 +56,28 @@ static { PERCENT_FORMAT.setMaximumFractionDigits(1); } - static final NumberFormat DOUBLE_FORMAT = NumberFormat.getNumberInstance(); + static final DecimalFormat DEVIATION_FORMAT = (DecimalFormat) NumberFormat.getPercentInstance(); static { - DOUBLE_FORMAT.setMaximumIntegerDigits(2); - DOUBLE_FORMAT.setMaximumFractionDigits(1); + DEVIATION_FORMAT.setMaximumFractionDigits(1); + DEVIATION_FORMAT.setMinimumFractionDigits(1); + DEVIATION_FORMAT.setPositivePrefix("+"); + DEVIATION_FORMAT.setNegativePrefix("- "); + } + static final DecimalFormat STDERR_FORMAT = (DecimalFormat) NumberFormat.getNumberInstance(); + static { + STDERR_FORMAT.setMaximumFractionDigits(1); + STDERR_FORMAT.setMinimumFractionDigits(1); + STDERR_FORMAT.setMultiplier(100); } public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD); - public final static String STANDARD_ERROR_MESSAGE="Standard error on this test is higher than "+STANDARD_ERROR_THRESHOLD_STRING; public final static String OK_IMAGE="OK.gif"; public final static String OK_IMAGE_WARN="OK_caution.gif"; public final static String FAIL_IMAGE="FAIL.gif"; public final static String FAIL_IMAGE_WARN="FAIL_caution.gif"; public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif"; public final static int OK = 0; - public final static int SIGN = 0x1; + public final static int WARN = 0x1; public final static int ERR = 0x2; -// public final static int TTEST = 0x2; public final static int DEV = 0x4; /** @@ -207,7 +212,7 @@ while (tokenizer.hasMoreTokens()) { String labelDescriptor = tokenizer.nextToken(); - String[] elements = labelDescriptor.split(","); + String[] elements = labelDescriptor.trim().split(","); ConfigDescriptor descriptor = new ConfigDescriptor(elements[0], elements[1]); configMap.put(elements[0], descriptor); } @@ -268,11 +273,11 @@ return componentNames; } - /** + /* * @param fp - * a FingerPrint object * @return - an html representation of the fingerprint. - */ + * public static String getImageMap(FingerPrint fp) { String componentDescription = fp.configDescriptor.description; String areas = fp.bar.getAreas(); @@ -287,6 +292,7 @@ } return output; } + */ /** * Utility method to copy a file. @@ -740,90 +746,86 @@ out.close(); } } - - public static double[] resultStats(Variations variations, String scenarioName, String baseline, String config) { - String OS = "config"; - - Variations tmpVariations=(Variations)variations.clone(); - tmpVariations.put(OS,config); - Scenario[] currentScenarios = DB.queryScenarios(tmpVariations, scenarioName,OS, null); - Variations referenceVariations = (Variations) variations.clone(); - referenceVariations.put(PerformanceTestPlugin.BUILD, baseline); - referenceVariations.put(OS, config); - Scenario[] refScenarios = DB.queryScenarios(referenceVariations, - scenarioName, OS, null); - - Map referenceScenariosMap = new HashMap(); - Map currentScenariosMap = new HashMap(); - for (int i = 0; i < refScenarios.length; i++) { - Scenario scenario = refScenarios[i]; - String name = scenario.getScenarioName(); - referenceScenariosMap.put(name, scenario); - } - - for (int i = 0; i < currentScenarios.length; i++) { - Scenario scenario = currentScenarios[i]; - String name = scenario.getScenarioName(); - currentScenariosMap.put(name, scenario); - } - Percentile percentile = StatisticsUtil.T90; - Scenario scenario = (Scenario) currentScenariosMap.get(scenarioName); - - Scenario reference = (Scenario) referenceScenariosMap.get(scenarioName); - if (reference != null) { - // XXX have to find out the relevant dimension - Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS; - TimeSeries currentSeries = scenario.getTimeSeries(significanceDimension); - TimeSeries baselineSeries = reference.getTimeSeries(significanceDimension); - if (currentSeries.getLength() > 0 && baselineSeries.getLength() > 0) { - return StatisticsUtil.statisticsForTimeSeries(baselineSeries, 0, currentSeries, 0, percentile); - } - } - return null; + + public static double[] resultsStatistics(TimeSeries timeSeries) { + try { + double valueRef = timeSeries.getValue(0), value = timeSeries.getValue(1); + long countRef = timeSeries.getCount(0), count = timeSeries.getCount(1); + double stddevRef = timeSeries.getStddev(0), stddev = timeSeries.getStddev(1); + double stderr = (countRef == 1 || count == 1) + ? Double.NaN + : (Double.isNaN(stddevRef) + ? Math.sqrt((stddev * stddev / count)) / valueRef + : Math.sqrt((stddevRef * stddevRef / countRef) + (stddev * stddev / count)) / valueRef); + return new double[] { + (value - valueRef) / valueRef, + stderr, + }; + } + catch (ArrayIndexOutOfBoundsException aioobe) { + return null; + } } - public static boolean hasConfidentResult(Variations variations, String scenarioName, String baseline, String config) { - double[] resultStats = resultStats(variations, scenarioName, baseline, config); + public static boolean hasConfidentResult(TimeSeries timeSeries) { + double[] resultStats = resultsStatistics(timeSeries); return (confidenceLevel(resultStats) & ERR) == 0; } public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) { - return failureMessage(resultStats(variations, scenarioName, baseline, config), true); + String current = (String) variations.get(PerformanceTestPlugin.BUILD); + Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS; + Scenario newScenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, baseline, current, new Dim[] { significanceDimension }); + TimeSeries timeSeries = newScenario.getTimeSeries(significanceDimension); + double[] results = resultsStatistics(timeSeries); + return failureMessage(results, true); } public static String failureMessage(double[] resultStats, boolean full) { StringBuffer buffer = new StringBuffer(); int level = confidenceLevel(resultStats); - boolean signal = (level & SIGN) != 0; +// boolean isWarn = (level & WARN) != 0; boolean isErr = (level & ERR) != 0; - if (full & isErr) { - buffer.append("*** WARNING *** "); - buffer.append(STANDARD_ERROR_MESSAGE); + if (full) { + if (isErr) { + buffer.append("*** WARNING *** "); + buffer.append(Messages.bind(Messages.standardError, PERCENT_FORMAT.format(resultStats[1]), STANDARD_ERROR_THRESHOLD_STRING)); + } + return buffer.toString(); } - if (!full) buffer.append(" "); if (resultStats != null) { - double deviation = resultStats[3]==0 ? 0 : -resultStats[3]; - if (deviation > 0) { - buffer.append('+'); - } - buffer.append(PERCENT_FORMAT.format(deviation)); - if (signal) { - buffer.append(" [±"); - buffer.append(DOUBLE_FORMAT.format(resultStats[2]*100)); - buffer.append(']'); + double deviation = resultStats[0]; + if (Double.isNaN(deviation) || Double.isInfinite(deviation)) { + buffer.append(" "); + buffer.append("[n/a]"); + } else { + double stderr = resultStats[1]; + deviation = Math.abs(deviation)<0.001 ? 0 : -deviation; + buffer.append(""); + if (Double.isNaN(stderr)) { + buffer.append(DEVIATION_FORMAT.format(deviation)); + buffer.append(" "); + buffer.append(" [n/a]"); + } else if (Double.isInfinite(stderr)) { + buffer.append(DEVIATION_FORMAT.format(deviation)); + buffer.append(" "); + buffer.append(" [Infinite]"); + } else { + buffer.append(DEVIATION_FORMAT.format(deviation)); + buffer.append(" [±"); + buffer.append(STDERR_FORMAT.format(Math.abs(stderr))); + buffer.append(']'); + } } + buffer.append(""); } - if (!full) buffer.append(""); return buffer.toString(); } public static int confidenceLevel(double[] resultStats) { int level = OK; if (resultStats != null){ -// if (resultStats[1] >= 0 && resultStats[0] >= resultStats[1]) { // invalid t-test -// level |= TTEST; -// } - if (resultStats[2] > 0) { // signal standard error higher than 0% (only one iteration) - level |= SIGN; + if (resultStats[1] >= (Utils.STANDARD_ERROR_THRESHOLD/2)) { // warns standard error higher than the half of authorized threshold + level |= WARN; } - if (resultStats[2] >= Utils.STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold + if (resultStats[1] >= Utils.STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold level |= ERR; } } @@ -837,15 +839,13 @@ String previous = ""; while (tokenizer.hasMoreTokens()) { String token = tokenizer.nextToken(); - if (token.equals("%")) { - start += previous.length(); - } else if (token.equals("_")) { - start++; - } else { + if (!token.equals("%") && !token.equals("_")) { if (previous.equals("%")) { - if (name.substring(start).indexOf(token) < 0) return false; + int idx = name.substring(start).indexOf(token); + if (idx < 0) return false; + start += idx; } else if (previous.equals("_")) { - if (!name.substring(start).startsWith(token)) return false; + if (!name.substring(++start).startsWith(token)) return false; } start += token.length(); } @@ -861,7 +861,7 @@ return name.equals(pattern); } - public static String getImage(int confidence, double[] resultStats, boolean hasExplanation) { + public static String getImage(int confidence, boolean hasExplanation) { boolean scenarioFailed = (confidence & DEV) != 0; String image = null; @@ -880,4 +880,73 @@ } return image; } -} + + public static boolean hasSummary(SummaryEntry[] summaries, String scenarioName) { + int length = summaries == null ? 0 : summaries.length; + for (int i=0; i= 0; + if (!hasClassName) { + testSeparator = scenarioName.lastIndexOf('.'); + if (testSeparator <= 0) { + if (max > 0 && scenarioName.length() > max) { + return "*"+scenarioName.substring(0, max); + } + return scenarioName; + } + } + int classSeparator = scenarioName.substring(0, testSeparator).lastIndexOf('.'); + if (classSeparator < 0) { + if (max > 0 && scenarioName.length() > max) { + return "*"+scenarioName.substring(0, max); + } + return scenarioName; + } + int length = scenarioName.length(); + String shortName = scenarioName.substring(classSeparator+1, length); + if (!hasClassName && shortName.startsWith("test.")) { // specific case for swt... + shortName = shortName.substring(5); + } + + // Remove qualification from test name + StringTokenizer tokenizer = new StringTokenizer(shortName, " :,", true); + StringBuffer buffer = new StringBuffer(tokenizer.nextToken()); + while (tokenizer.hasMoreTokens()) { + String token = tokenizer.nextToken(); + char fc = token.charAt(0); + while (fc == ' ' || fc == ',' || fc == ':') { + buffer.append(token); // add the separator + token = tokenizer.nextToken(); + fc = token.charAt(0); + } + int last = token .lastIndexOf('.'); + if (last >= 3) { + int first = token .indexOf('.'); + if (first == last) { + buffer.append(token); + } else { +// buffer.append(token.substring(0, first)); +// buffer.append("..."); + buffer.append(token.substring(last+1)); + } + } else { + buffer.append(token); + } + } + if (max > 0 && buffer.length() > max) { + return "*"+buffer.substring(0, max); + } + return buffer.toString(); + } +} \ No newline at end of file Index: src/org/eclipse/test/performance/ui/ScenarioStatusTable.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/ScenarioStatusTable.java,v retrieving revision 1.28 diff -u -r1.28 ScenarioStatusTable.java --- src/org/eclipse/test/performance/ui/ScenarioStatusTable.java 15 May 2007 19:23:10 -0000 1.28 +++ src/org/eclipse/test/performance/ui/ScenarioStatusTable.java 31 May 2007 16:46:33 -0000 @@ -10,12 +10,19 @@ *******************************************************************************/ package org.eclipse.test.performance.ui; +import java.io.PrintStream; import java.util.ArrayList; import java.util.Hashtable; +import java.util.StringTokenizer; +import org.eclipse.test.internal.performance.PerformanceTestPlugin; +import org.eclipse.test.internal.performance.data.Dim; import org.eclipse.test.internal.performance.db.DB; import org.eclipse.test.internal.performance.db.Scenario; +import org.eclipse.test.internal.performance.db.SummaryEntry; +import org.eclipse.test.internal.performance.db.TimeSeries; import org.eclipse.test.internal.performance.db.Variations; +import org.eclipse.test.performance.Dimension; public class ScenarioStatusTable { @@ -24,18 +31,23 @@ private String scenarioPattern; private ArrayList configNames=new ArrayList(); private Hashtable scenarioComments; + private SummaryEntry[] fingerprintEntries; private final String baseline; private class ScenarioStatus{ Hashtable statusMap; - String name; + String name, shortName; Hashtable configStatus; + Hashtable resultsMap; boolean hasSlowDownExplanation=false; + boolean fingerprint = false; + boolean hasBaseline = true; public ScenarioStatus(String scenarioName){ - name=scenarioName; - statusMap=new Hashtable(); - configStatus=new Hashtable(); + name = scenarioName; + statusMap = new Hashtable(); + configStatus = new Hashtable(); + resultsMap = new Hashtable(); } } @@ -48,20 +60,20 @@ * @param scenarioPattern * @param configDescriptors */ - public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, String baseline){ + public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, SummaryEntry[] fpSummaries, String baseline){ configMaps=configDescriptors; this.variations=variations; this.scenarioPattern=scenarioPattern; this.scenarioComments=scenarioComments; this.baseline= baseline; + this.fingerprintEntries = fpSummaries; } - + /** - * Returns HTML representation of scenario status table. + * Prints the HTML representation of scenario status table into the given stream. */ - public String toString() { + public void print(PrintStream stream, boolean filter) { String OS="config"; - String htmlTable=""; Scenario[] scenarios= DB.queryScenarios(variations, scenarioPattern, OS, null); if (scenarios != null && scenarios.length > 0) { @@ -70,23 +82,35 @@ for (int i= 0; i < scenarios.length; i++) { Scenario scenario= scenarios[i]; String scenarioName=scenario.getScenarioName(); -// if (!Utils.matchPattern(scenarioName, scenarioPattern)) continue; + if (filter && !Utils.matchPattern(scenarioName, scenarioPattern)) continue; // returns the config names. Making assumption that indices in // the configs array map to the indices of the failure messages. String[] configs=scenario.getTimeSeriesLabels(); String[] failureMessages= scenario.getFailureMessages(); ScenarioStatus scenarioStatus=new ScenarioStatus(scenarioName); - scenarioStatusList.add(scenarioStatus); + scenarioStatus.fingerprint = Utils.hasSummary(this.fingerprintEntries, scenarioName); String scenarioComment= (String)scenarioComments.get(scenarioName); if (scenarioComment != null) scenarioStatus.hasSlowDownExplanation= true; - - for (int j=0;j(vs. "); + shortName.append(timeSeriesLabels[0]); + shortName.append(")"); + scenarioStatus.shortName = shortName.toString(); + } + } + + if (!scenarioStatusList.contains(scenarioStatus)) { + scenarioStatusList.add(scenarioStatus); + } } } String label=null; - htmlTable=htmlTable.concat("

Scenario Status

\n" + - "The scenario status table shows all scenarios tests result for all performance test machines. It gives a complete but compact view of performance result for the component.
\n" + - "For each test (ie. in each cell of this table), following information are displayed:\n" + - "
    \n" + - "
  • an icon showing whether the test fails or passes and whether it's reliable or not.
    \n" + - "The legend for this icon is:\n" + - "
      \n" + - "
    • Green (): mark a successful result, which means this test has neither significant performance regression nor significant standard error
    • \n" + - "
    • Red (): mark a failing result, which means this test shows a significant performance regression (more than 10%)
    • \n" + - "
    • Gray (): mark a failing result (see above) with a comment explaining this degradation.
    • \n" + - "
    • Yellow ( or ): mark a failing or successful result with a significant standard error (more than "+Utils.STANDARD_ERROR_THRESHOLD_STRING+")
    • \n" + - "
    • \"n/a\": mark a test for with no performance results
    • \n" + - "
  • \n" + - "
  • the value of the deviation from the baseline as a percentage (ie. formula is: (build_test_time - baseline_test_time) / baseline_test_time)
  • \n" + - "
  • the value of the standard error of this deviation as a percentage (ie. formula is: sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time)
    \n" + - "Note that errors equal to 0 are not shown (tests which have only one iteration).
  • \n" + - "
" + - "For failing tests, value of deviation with its standard error is added at the beginning of the error message you can see flying over the corresponding image.
\n" + - "Follow the link on test box corresponding image for detailed results.
" + - "
\n"); + stream.println("

Scenario Status

"); + stream.println("The scenario status table shows all scenarios tests result for all performance test machines."); + stream.println("It gives a complete but compact view of performance result for the component.
"); + stream.println("For each test (ie. in each cell of this table), following information are displayed:"); + stream.println("
    "); + stream.println("
  • an icon showing whether the test fails or passes and whether it's reliable or not.
    "); + stream.println("The legend for this icon is:"); + stream.println("
      "); + stream.print("
    • Green (): mark a successful result, which means this test has neither significant performance regression nor significant standard error
    • "); + stream.print("
    • Red (): mark a failing result, which means this test shows a significant performance regression (more than 10%)
    • "); + stream.print("
    • Gray (): mark a failing result (see above) with a comment explaining this degradation.
    • "); + stream.print("
    • Yellow ( or ): mark a failing or successful result with a significant standard error (more than "); + stream.print(Utils.STANDARD_ERROR_THRESHOLD_STRING); + stream.println(")
    • "); + stream.println("
    • \"n/a\": mark a test for with no performance results
    • "); + stream.println("
  • "); + stream.println("
  • the value of the deviation from the baseline as a percentage (ie. formula is: (build_test_time - baseline_test_time) / baseline_test_time)
    "); + stream.println("This value may be '[n/a]' when deviation is not a number (NaN) or is infinite, typically when reference value is equals to 0!
  • "); + stream.println("
  • the value of the standard error of this deviation as a percentage (ie. formula is: sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time)
    "); + stream.println("This value may be '[n/a]' when error cannot be computed, typically when the test has only one measure!
  • "); + stream.println("
"); + stream.println("Hints:
    "); + stream.println("
  • fly over image of failing tests to see the complete error message
  • "); + stream.println("
  • to look at the complete and detailed test results, click on its image
  • "); + stream.println("
  • scenario name may be emphazised when the results are also displayed in the fingerprint
  • "); + stream.println("
  • scenario name may start with an '*' when the scenario has no results in the last baseline run
  • "); + stream.println("
"); + stream.println(); + stream.println(""); + stream.println(""); + stream.print(""); - htmlTable=htmlTable.concat("

All "); + stream.print(scenarios.length); + stream.println(" scenarios

\n"); for (int i= 0; i < configNames.size(); i++){ - label=configNames.get(i).toString(); - String columnTitle=label; + label = configNames.get(i).toString(); + String columnTitle = label; if (configMaps!=null) { Utils.ConfigDescriptor configDescriptor= (Utils.ConfigDescriptor)configMaps.get(label); - if (configDescriptor != null) - columnTitle=configDescriptor.description; + if (configDescriptor != null) { + int idx = configDescriptor.description.indexOf('('); + if (idx < 0) { + columnTitle = configDescriptor.description; + } else { + // first line + StringTokenizer tokenizer = new StringTokenizer(configDescriptor.description.substring(0, idx).trim(), " "); + StringBuffer buffer = new StringBuffer(tokenizer.nextToken()); + while (tokenizer.hasMoreTokens()) { + buffer.append(" "); + buffer.append(tokenizer.nextToken()); + } + buffer.append(' '); + // second line + tokenizer = new StringTokenizer(configDescriptor.description.substring(idx).trim(), " "); + buffer.append(tokenizer.nextToken()); + while (tokenizer.hasMoreTokens()) { + buffer.append(" "); + buffer.append(tokenizer.nextToken()); + } + columnTitle = buffer.toString(); + } + } } - htmlTable=htmlTable.concat(""); + stream.print("\n"); // counter for js class Id's int jsIdCount=0; @@ -147,61 +233,63 @@ ScenarioStatus status=(ScenarioStatus)scenarioStatusList.get(j); - htmlTable=htmlTable.concat(""); + stream.println(""); + stream.print(""); - htmlTable=htmlTable.concat(html.toString()); + String results = (String) status.resultsMap.get(configName); + if (results != null) stream.println(results); }else{ - htmlTable=htmlTable.concat(""); + stream.println("\n"); + stream.println("

All "+scenarios.length+" scenarios

"+columnTitle +"
"); + stream.print(columnTitle); + stream.println("
"); } - - htmlTable=htmlTable.concat("
"+status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1)+"
"); + if (status.fingerprint) stream.print(""); + if (!status.hasBaseline) stream.print("*"); +// stream.print(status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1)); + stream.print(status.shortName); + if (!status.hasBaseline) stream.print(""); + if (status.fingerprint) stream.print(""); + stream.println(); for (int i=0;i\n"); + stream.print("href=\""); + stream.print(aUrl); + stream.print('/'); + stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_')); + stream.println(".html\">"); + stream.print(""); } else { // create message with tooltip text including deviation with error plus failure message jsIdCount+=1; - html.append("class=\"tooltipSource\" onMouseover=\"show_element('toolTip"); - html.append(jsIdCount); - html.append("')\" onMouseout=\"hide_element('toolTip"); - html.append(jsIdCount); - html.append("')\" \nhref=\""); - html.append(aUrl); - html.append('/'); - html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_')); - html.append(".html\">\n\n"); - html.append(message); - html.append(""); + stream.print("class=\"tooltipSource\" onMouseover=\"show_element('toolTip"); + stream.print(jsIdCount); + stream.print("')\" onMouseout=\"hide_element('toolTip"); + stream.print(jsIdCount); + stream.print("')\" \nhref=\""); + stream.print(aUrl); + stream.print('/'); + stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_')); + stream.println(".html\">"); + stream.print(""); + stream.print(""); + stream.print(message); + stream.println(""); } - html.append(Utils.failureMessage(resultStats, false)); - html.append("n/an/a"); } } + stream.flush(); } - - htmlTable=htmlTable.concat("
"); } - return htmlTable; - } + } } Index: src/org/eclipse/test/performance/ui/FingerPrint.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/FingerPrint.java,v retrieving revision 1.20 diff -u -r1.20 FingerPrint.java --- src/org/eclipse/test/performance/ui/FingerPrint.java 27 Apr 2007 19:07:44 -0000 1.20 +++ src/org/eclipse/test/performance/ui/FingerPrint.java 31 May 2007 16:46:33 -0000 @@ -19,17 +19,11 @@ import java.util.Hashtable; import org.eclipse.swt.SWT; -import org.eclipse.swt.events.PaintEvent; -import org.eclipse.swt.events.PaintListener; import org.eclipse.swt.graphics.GC; import org.eclipse.swt.graphics.Image; import org.eclipse.swt.graphics.ImageData; import org.eclipse.swt.graphics.ImageLoader; -import org.eclipse.swt.graphics.Point; -import org.eclipse.swt.layout.FillLayout; -import org.eclipse.swt.widgets.Canvas; import org.eclipse.swt.widgets.Display; -import org.eclipse.swt.widgets.Shell; import org.eclipse.test.internal.performance.PerformanceTestPlugin; import org.eclipse.test.internal.performance.data.Dim; import org.eclipse.test.internal.performance.db.DB; @@ -37,9 +31,6 @@ import org.eclipse.test.internal.performance.db.SummaryEntry; import org.eclipse.test.internal.performance.db.TimeSeries; import org.eclipse.test.internal.performance.db.Variations; -import org.eclipse.test.internal.performance.eval.StatisticsUtil; -import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile; -import org.eclipse.test.performance.Dimension; import org.eclipse.test.performance.ui.Utils.ConfigDescriptor; @@ -74,17 +65,16 @@ if (component==null){ entries= DB.querySummaries(variations,null); this.component=""; - } - else + } else { entries=DB.querySummaries(variations,component+'%'); - run(entries); + } + run(); } /** * Creates the fingerprint gif, image map and scenario status table for the component. - * @param entries - the result of a database query for summaries for a specified variation. */ - public void run(Object[] entries) { + public void run() { new File(outputDirectory).mkdirs(); String referenceName=referenceBuildId; String currentName=currentBuildId; @@ -101,12 +91,12 @@ if (entries != null) { for (int i= 0; i < entries.length; i++) { - SummaryEntry se= (SummaryEntry)entries[i]; - if (se.comment==null) - add(bar, se.shortName, new Dim[] { se.dimension }, se.scenarioName); + SummaryEntry summary = entries[i]; + if (summary.comment==null) + add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName); else{ - setComment(se.scenarioName, se.comment); - add(bar, se.shortName, new Dim[] { se.dimension }, se.scenarioName,se.comment); + setComment(summary.scenarioName, summary.comment); + add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName, summary.comment); } } } @@ -115,7 +105,7 @@ if (component=="") outName= "FP_"+referenceName + '_' + currentBuildId+"."+configDescriptor.name; - save(bar, outputDirectory + '/' + outName); + save(outputDirectory + '/' + outName); //show(bar); @@ -127,11 +117,11 @@ scenarioComments.put(scenario,comment); } - private void add(BarGraph bar, String name, Dim[] dims, String scenarioName) { - add (bar,name,dims,scenarioName,null); + private void add(String name, Dim[] dims, String scenarioName) { + add (name,dims,scenarioName,null); } - private void add(BarGraph bar, String name, Dim[] dims, String scenarioName,String comment) { + private void add(String name, Dim[] dims, String scenarioName, String comment) { String refData= ""; Scenario scenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, referenceBuildId, currentBuildId, dims); String[] timeSeriesLabels= scenario.getTimeSeriesLabels(); @@ -153,7 +143,8 @@ double percent= 0.0; boolean hasConfidentResult= true; if (l > 1) { - hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name); +// hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name); + hasConfidentResult= Utils.hasConfidentResult(timeSeries); /*if (!rejectNullHypothesis) { NumberFormat percentFormatter= NumberFormat.getPercentInstance(); String statisticsComment= "There is not enough evidence to reject the null hypothesis at the " + percentFormatter.format(percentile.inside()) + "level"; @@ -178,7 +169,7 @@ } - private void save(BarGraph bar, String output) { + private void save(String output) { //if (bar.getFItems().size()==0) //return; @@ -217,7 +208,7 @@ /* * Displays bar graph in window - */ + * private void show(final BarGraph bar) { Display display= new Display(); @@ -246,4 +237,23 @@ public String getOutName() { return outName; } + */ + + /** + * @return - an html representation of the fingerprint. + */ + public String getImageMap() { + String componentDescription = this.configDescriptor.description; + String areas = this.bar.getAreas(); + if (areas == null) + areas = ""; + String output = ""; + if (new File(this.outputDirectory, this.outName + ".gif").exists()) { + output = "

" + componentDescription + "

"; + output = output.concat("" + "" + areas + "\n"); + } else { + output = output.concat("

There is no fingerprint for " + componentDescription + "

\n"); + } + return output; + } } Index: src/org/eclipse/test/performance/ui/ScenarioResults.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/ScenarioResults.java,v retrieving revision 1.32 diff -u -r1.32 ScenarioResults.java --- src/org/eclipse/test/performance/ui/ScenarioResults.java 15 May 2007 19:23:10 -0000 1.32 +++ src/org/eclipse/test/performance/ui/ScenarioResults.java 31 May 2007 16:46:33 -0000 @@ -15,7 +15,6 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.PrintStream; -import java.text.NumberFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.Hashtable; @@ -48,7 +47,7 @@ * * @param scenarios - * the array of Scenario objects for which to generate results. - * @param reference - + * @param baseline - * the reference build ID * @param current - * the current buildID Index: src/org/eclipse/test/performance/ui/Main.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/Main.java,v retrieving revision 1.40 diff -u -r1.40 Main.java --- src/org/eclipse/test/performance/ui/Main.java 15 May 2007 19:23:10 -0000 1.40 +++ src/org/eclipse/test/performance/ui/Main.java 31 May 2007 16:46:33 -0000 @@ -25,6 +25,7 @@ import org.eclipse.core.runtime.Platform; import org.eclipse.test.internal.performance.db.Scenario; +import org.eclipse.test.internal.performance.db.SummaryEntry; import org.eclipse.test.internal.performance.db.Variations; import org.eclipse.test.performance.ui.Utils.ConfigDescriptor; @@ -48,14 +49,21 @@ private Hashtable fingerPrints = new Hashtable(); private Hashtable scenarioComments=new Hashtable(); private Hashtable rawDataTables=new Hashtable(); + private boolean local = false; public Object run(Object args) throws Exception { parse(args); - Enumeration configIds=configDescriptors.keys(); - - while (configIds.hasMoreElements()){ - generate((ConfigDescriptor)configDescriptors.get(configIds.nextElement())); + if (this.local) { + int length = configNames.length; + for (int i=0; i"); Hashtable fps = (Hashtable) fingerPrints.get(component); - Enumeration configs = fps.keys(); int baselineUnderScoreIndex=baseline.indexOf("_"); int currentUnderScoreIndex=currentBuildId.indexOf("_"); @@ -106,17 +113,24 @@ os.println(title); //print the html representation of fingerprint for each config + Enumeration configs = fps.keys(); + SummaryEntry[] fpSummaries = null; while (configs.hasMoreElements()) { String config = configs.nextElement().toString(); - FingerPrint fp = (FingerPrint) fps.get(config); - os.println(Utils.getImageMap(fp)); + FingerPrint fingerPrint = (FingerPrint) fps.get(config); + os.println(fingerPrint.getImageMap()); + if (fpSummaries == null) { + fpSummaries = fingerPrint.entries; + } } - if (component != "") { + if (component.length() > 0) { //print the component scenario status table beneath the fingerprint variations.put("config", "%"); - ScenarioStatusTable sst = new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, baseline); -// ScenarioStatusTable sst = new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, baseline); - os.println(sst.toString()); + boolean filter = this.local & this.scenarioFilter != null; // use scenario filter to minimize DB requests while testing... + ScenarioStatusTable sst = filter + ? new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, fpSummaries, baseline) + : new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, fpSummaries, baseline); + sst.print(os, filter); } os.println(Utils.HTML_CLOSE); @@ -292,8 +306,7 @@ System.out.println("Missing value for -config.properties parameter"); printUsage(); } - configDescriptors = Utils - .getConfigDescriptors(configProperties); + configDescriptors = Utils.getConfigDescriptors(configProperties); i++; continue; } @@ -318,6 +331,11 @@ i++; continue; } + if (arg.equals("-local")) { + this.local = true; + i++; + continue; + } i++; } Index: src/org/eclipse/test/performance/ui/messages.properties =================================================================== RCS file: src/org/eclipse/test/performance/ui/messages.properties diff -N src/org/eclipse/test/performance/ui/messages.properties --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/org/eclipse/test/performance/ui/messages.properties 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,14 @@ +############################################################################### +# Copyright (c) 2000, 2007 IBM Corporation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# Contributors: +# IBM Corporation - initial API and implementation +############################################################################### + +### Performance Tests Messages. + +standardError = Standard error on this test is {0} (should be less than {1} to become reliable!) Index: src/org/eclipse/test/performance/ui/Messages.java =================================================================== RCS file: src/org/eclipse/test/performance/ui/Messages.java diff -N src/org/eclipse/test/performance/ui/Messages.java --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/org/eclipse/test/performance/ui/Messages.java 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (c) 2000, 2007 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ +package org.eclipse.test.performance.ui; + +import java.text.MessageFormat; + +import org.eclipse.osgi.util.NLS; + +public class Messages extends NLS { + + private static final String BUNDLE_NAME = "org.eclipse.test.performance.ui.messages";//$NON-NLS-1$ + + private Messages() { + // Do not instantiate + } + + public static String standardError; + + static { + NLS.initializeMessages(BUNDLE_NAME, Messages.class); + } + + /** + * Bind the given message's substitution locations with the given string values. + * + * @param message the message to be manipulated + * @return the manipulated String + */ + public static String bind(String message) { + return bind(message, null); + } + + /** + * Bind the given message's substitution locations with the given string values. + * + * @param message the message to be manipulated + * @param binding the object to be inserted into the message + * @return the manipulated String + */ + public static String bind(String message, Object binding) { + return bind(message, new Object[] {binding}); + } + + /** + * Bind the given message's substitution locations with the given string values. + * + * @param message the message to be manipulated + * @param binding1 An object to be inserted into the message + * @param binding2 A second object to be inserted into the message + * @return the manipulated String + */ + public static String bind(String message, Object binding1, Object binding2) { + return bind(message, new Object[] {binding1, binding2}); + } + + /** + * Bind the given message's substitution locations with the given string values. + * + * @param message the message to be manipulated + * @param bindings An array of objects to be inserted into the message + * @return the manipulated String + */ + public static String bind(String message, Object[] bindings) { + return MessageFormat.format(message, bindings); + } + +}