### Eclipse Workspace Patch 1.0
#P org.eclipse.test.performance.ui
Index: src/org/eclipse/test/performance/ui/Utils.java
===================================================================
RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/Utils.java,v
retrieving revision 1.34
diff -u -r1.34 Utils.java
--- src/org/eclipse/test/performance/ui/Utils.java 15 May 2007 19:23:10 -0000 1.34
+++ src/org/eclipse/test/performance/ui/Utils.java 1 Jun 2007 16:28:56 -0000
@@ -20,13 +20,13 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
+import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Hashtable;
-import java.util.Map;
import java.util.StringTokenizer;
import junit.framework.AssertionFailedError;
@@ -43,10 +43,9 @@
import org.eclipse.test.internal.performance.data.Dim;
import org.eclipse.test.internal.performance.db.DB;
import org.eclipse.test.internal.performance.db.Scenario;
+import org.eclipse.test.internal.performance.db.SummaryEntry;
import org.eclipse.test.internal.performance.db.TimeSeries;
import org.eclipse.test.internal.performance.db.Variations;
-import org.eclipse.test.internal.performance.eval.StatisticsUtil;
-import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
import org.eclipse.test.performance.Dimension;
@@ -57,22 +56,29 @@
static {
PERCENT_FORMAT.setMaximumFractionDigits(1);
}
- static final NumberFormat DOUBLE_FORMAT = NumberFormat.getNumberInstance();
+ static final DecimalFormat DEVIATION_FORMAT = (DecimalFormat) NumberFormat.getPercentInstance();
static {
- DOUBLE_FORMAT.setMaximumIntegerDigits(2);
- DOUBLE_FORMAT.setMaximumFractionDigits(1);
+ DEVIATION_FORMAT.setMaximumFractionDigits(1);
+ DEVIATION_FORMAT.setMinimumFractionDigits(1);
+ DEVIATION_FORMAT.setPositivePrefix("+");
+ DEVIATION_FORMAT.setNegativePrefix("- ");
+ }
+ static final DecimalFormat STDERR_FORMAT = (DecimalFormat) NumberFormat.getNumberInstance();
+ static {
+ STDERR_FORMAT.setMaximumFractionDigits(1);
+ STDERR_FORMAT.setMinimumFractionDigits(1);
+ STDERR_FORMAT.setMultiplier(100);
}
public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);
- public final static String STANDARD_ERROR_MESSAGE="Standard error on this test is higher than "+STANDARD_ERROR_THRESHOLD_STRING;
+ public final static String UNKNOWN_IMAGE="Unknown.gif";
public final static String OK_IMAGE="OK.gif";
public final static String OK_IMAGE_WARN="OK_caution.gif";
public final static String FAIL_IMAGE="FAIL.gif";
public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";
public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";
public final static int OK = 0;
- public final static int SIGN = 0x1;
+ public final static int NAN = 0x1;
public final static int ERR = 0x2;
-// public final static int TTEST = 0x2;
public final static int DEV = 0x4;
/**
@@ -207,7 +213,7 @@
while (tokenizer.hasMoreTokens()) {
String labelDescriptor = tokenizer.nextToken();
- String[] elements = labelDescriptor.split(",");
+ String[] elements = labelDescriptor.trim().split(",");
ConfigDescriptor descriptor = new ConfigDescriptor(elements[0], elements[1]);
configMap.put(elements[0], descriptor);
}
@@ -268,11 +274,11 @@
return componentNames;
}
- /**
+ /*
* @param fp -
* a FingerPrint object
* @return - an html representation of the fingerprint.
- */
+ *
public static String getImageMap(FingerPrint fp) {
String componentDescription = fp.configDescriptor.description;
String areas = fp.bar.getAreas();
@@ -287,6 +293,7 @@
}
return output;
}
+ */
/**
* Utility method to copy a file.
@@ -314,11 +321,12 @@
}
}
public static void copyImages(File images, File output) {
- copyFile(new File(images, Utils.FAIL_IMAGE), new File(output, Utils.FAIL_IMAGE));
- copyFile(new File(images, Utils.FAIL_IMAGE_EXPLAINED), new File(output, Utils.FAIL_IMAGE_EXPLAINED));
- copyFile(new File(images, Utils.FAIL_IMAGE_WARN), new File(output, Utils.FAIL_IMAGE_WARN));
- copyFile(new File(images, Utils.OK_IMAGE), new File(output, Utils.OK_IMAGE));
- copyFile(new File(images, Utils.OK_IMAGE_WARN), new File(output, Utils.OK_IMAGE_WARN));
+ copyFile(new File(images, FAIL_IMAGE), new File(output, FAIL_IMAGE));
+ copyFile(new File(images, FAIL_IMAGE_EXPLAINED), new File(output, FAIL_IMAGE_EXPLAINED));
+ copyFile(new File(images, FAIL_IMAGE_WARN), new File(output, FAIL_IMAGE_WARN));
+ copyFile(new File(images, OK_IMAGE), new File(output, OK_IMAGE));
+ copyFile(new File(images, OK_IMAGE_WARN), new File(output, OK_IMAGE_WARN));
+ copyFile(new File(images, UNKNOWN_IMAGE), new File(output, UNKNOWN_IMAGE));
}
public static void copyScripts(File scripts, File output) {
copyFile(new File(scripts, "ToolTip.css"), new File(output, "ToolTip.css"));
@@ -433,7 +441,7 @@
p.paint(image);
/* Downscale to 8 bit depth palette to save to gif */
- ImageData data = Utils.downSample(image);
+ ImageData data = downSample(image);
ImageLoader il = new ImageLoader();
il.data = new ImageData[] { data };
OutputStream out = null;
@@ -599,7 +607,7 @@
* @return date/time in format YYYYMMDDHHMM, ie. 200504060010
*/
public static long getDateFromBuildID(String buildId) {
- return Utils.getDateFromBuildID(buildId, false);
+ return getDateFromBuildID(buildId, false);
}
public static long getDateFromBuildID(String buildId, boolean matchLast) {
@@ -656,7 +664,7 @@
PrintWriter out=null;
try {
out = new PrintWriter(new FileWriter(new File(outputFile)));
- out.println(Utils.HTML_OPEN + "
\n");
+ out.println(HTML_OPEN + "\n");
out.println("Summary of Elapsed Process Variation Coefficients
\n"+
" This table provides a bird's eye view of variability in elapsed process times\n"+
"for baseline and current build stream performance scenarios." +
@@ -740,91 +748,86 @@
out.close();
}
}
-
- public static double[] resultStats(Variations variations, String scenarioName, String baseline, String config) {
- String OS = "config";
-
- Variations tmpVariations=(Variations)variations.clone();
- tmpVariations.put(OS,config);
- Scenario[] currentScenarios = DB.queryScenarios(tmpVariations, scenarioName,OS, null);
- Variations referenceVariations = (Variations) variations.clone();
- referenceVariations.put(PerformanceTestPlugin.BUILD, baseline);
- referenceVariations.put(OS, config);
- Scenario[] refScenarios = DB.queryScenarios(referenceVariations,
- scenarioName, OS, null);
-
- Map referenceScenariosMap = new HashMap();
- Map currentScenariosMap = new HashMap();
- for (int i = 0; i < refScenarios.length; i++) {
- Scenario scenario = refScenarios[i];
- String name = scenario.getScenarioName();
- referenceScenariosMap.put(name, scenario);
- }
-
- for (int i = 0; i < currentScenarios.length; i++) {
- Scenario scenario = currentScenarios[i];
- String name = scenario.getScenarioName();
- currentScenariosMap.put(name, scenario);
- }
- Percentile percentile = StatisticsUtil.T90;
- Scenario scenario = (Scenario) currentScenariosMap.get(scenarioName);
-
- Scenario reference = (Scenario) referenceScenariosMap.get(scenarioName);
- if (reference != null) {
- // XXX have to find out the relevant dimension
- Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;
- TimeSeries currentSeries = scenario.getTimeSeries(significanceDimension);
- TimeSeries baselineSeries = reference.getTimeSeries(significanceDimension);
- if (currentSeries.getLength() > 0 && baselineSeries.getLength() > 0) {
- return StatisticsUtil.statisticsForTimeSeries(baselineSeries, 0, currentSeries, 0, percentile);
- }
- }
- return null;
+
+ public static double[] resultsStatistics(TimeSeries timeSeries) {
+ try {
+ double valueRef = timeSeries.getValue(0), value = timeSeries.getValue(1);
+ long countRef = timeSeries.getCount(0), count = timeSeries.getCount(1);
+ double stddevRef = timeSeries.getStddev(0), stddev = timeSeries.getStddev(1);
+ double stderr = (countRef == 1 || count == 1)
+ ? Double.NaN
+ : (Double.isNaN(stddevRef)
+ ? Math.sqrt((stddev * stddev / count)) / valueRef
+ : Math.sqrt((stddevRef * stddevRef / countRef) + (stddev * stddev / count)) / valueRef);
+ return new double[] {
+ (value - valueRef) / valueRef,
+ stderr,
+ };
+ }
+ catch (ArrayIndexOutOfBoundsException aioobe) {
+ return null;
+ }
}
- public static boolean hasConfidentResult(Variations variations, String scenarioName, String baseline, String config) {
- double[] resultStats = resultStats(variations, scenarioName, baseline, config);
+ public static boolean hasConfidentResult(TimeSeries timeSeries) {
+ double[] resultStats = resultsStatistics(timeSeries);
return (confidenceLevel(resultStats) & ERR) == 0;
}
public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) {
- return failureMessage(resultStats(variations, scenarioName, baseline, config), true);
+ String current = (String) variations.get(PerformanceTestPlugin.BUILD);
+ Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;
+ Scenario newScenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, baseline, current, new Dim[] { significanceDimension });
+ TimeSeries timeSeries = newScenario.getTimeSeries(significanceDimension);
+ double[] results = resultsStatistics(timeSeries);
+ return failureMessage(results, true);
}
public static String failureMessage(double[] resultStats, boolean full) {
StringBuffer buffer = new StringBuffer();
int level = confidenceLevel(resultStats);
- boolean signal = (level & SIGN) != 0;
+// boolean isWarn = (level & WARN) != 0;
boolean isErr = (level & ERR) != 0;
- if (full & isErr) {
- buffer.append("*** WARNING *** ");
- buffer.append(STANDARD_ERROR_MESSAGE);
+ if (full) {
+ if (isErr) {
+ buffer.append("*** WARNING *** ");
+ buffer.append(Messages.bind(Messages.standardError, PERCENT_FORMAT.format(resultStats[1]), STANDARD_ERROR_THRESHOLD_STRING));
+ }
+ return buffer.toString();
}
- if (!full) buffer.append(" ");
if (resultStats != null) {
- double deviation = resultStats[3]==0 ? 0 : -resultStats[3];
- if (deviation > 0) {
- buffer.append('+');
- }
- buffer.append(PERCENT_FORMAT.format(deviation));
- if (signal) {
- buffer.append(" [±");
- buffer.append(DOUBLE_FORMAT.format(resultStats[2]*100));
- buffer.append(']');
+ double deviation = resultStats[0];
+ buffer.append("");
+ if (Double.isNaN(deviation) || Double.isInfinite(deviation)) {
+ buffer.append(" [n/a]");
+ } else {
+ double stderr = resultStats[1];
+ deviation = Math.abs(deviation)<0.001 ? 0 : -deviation;
+ if (Double.isNaN(stderr) || Double.isInfinite(stderr)) {
+ buffer.append(DEVIATION_FORMAT.format(deviation));
+ buffer.append(" ");
+ buffer.append(" [n/a]");
+ } else {
+ buffer.append(DEVIATION_FORMAT.format(deviation));
+ buffer.append(" [±");
+ buffer.append(STDERR_FORMAT.format(Math.abs(stderr)));
+ buffer.append(']');
+ }
}
+ buffer.append("");
}
- if (!full) buffer.append("");
return buffer.toString();
}
public static int confidenceLevel(double[] resultStats) {
int level = OK;
if (resultStats != null){
-// if (resultStats[1] >= 0 && resultStats[0] >= resultStats[1]) { // invalid t-test
-// level |= TTEST;
-// }
- if (resultStats[2] > 0) { // signal standard error higher than 0% (only one iteration)
- level |= SIGN;
- }
- if (resultStats[2] >= Utils.STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold
- level |= ERR;
+ if (Double.isNaN(resultStats[0]) || Double.isInfinite(resultStats[0])) {
+ level = NAN;
+ } else {
+// if (resultStats[1] >= (STANDARD_ERROR_THRESHOLD/2)) { // warns standard error higher than the half of authorized threshold
+// level |= WARN;
+// }
+ if (resultStats[1] >= STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold
+ level = ERR;
+ }
}
}
return level;
@@ -837,15 +840,13 @@
String previous = "";
while (tokenizer.hasMoreTokens()) {
String token = tokenizer.nextToken();
- if (token.equals("%")) {
- start += previous.length();
- } else if (token.equals("_")) {
- start++;
- } else {
+ if (!token.equals("%") && !token.equals("_")) {
if (previous.equals("%")) {
- if (name.substring(start).indexOf(token) < 0) return false;
+ int idx = name.substring(start).indexOf(token);
+ if (idx < 0) return false;
+ start += idx;
} else if (previous.equals("_")) {
- if (!name.substring(start).startsWith(token)) return false;
+ if (!name.substring(++start).startsWith(token)) return false;
}
start += token.length();
}
@@ -861,7 +862,7 @@
return name.equals(pattern);
}
- public static String getImage(int confidence, double[] resultStats, boolean hasExplanation) {
+ public static String getImage(int confidence, boolean hasExplanation) {
boolean scenarioFailed = (confidence & DEV) != 0;
String image = null;
@@ -873,6 +874,8 @@
} else {
image = FAIL_IMAGE;
}
+ } else if ((confidence & NAN) != 0) {
+ image = UNKNOWN_IMAGE;
} else if ((confidence & ERR) != 0) {
image = OK_IMAGE_WARN;
} else {
@@ -880,4 +883,73 @@
}
return image;
}
-}
+
+ public static boolean hasSummary(SummaryEntry[] summaries, String scenarioName) {
+ int length = summaries == null ? 0 : summaries.length;
+ for (int i=0; i= 0;
+ if (!hasClassName) {
+ testSeparator = scenarioName.lastIndexOf('.');
+ if (testSeparator <= 0) {
+ if (max > 0 && scenarioName.length() > max) {
+ return "*"+scenarioName.substring(0, max);
+ }
+ return scenarioName;
+ }
+ }
+ int classSeparator = scenarioName.substring(0, testSeparator).lastIndexOf('.');
+ if (classSeparator < 0) {
+ if (max > 0 && scenarioName.length() > max) {
+ return "*"+scenarioName.substring(0, max);
+ }
+ return scenarioName;
+ }
+ int length = scenarioName.length();
+ String shortName = scenarioName.substring(classSeparator+1, length);
+ if (!hasClassName && shortName.startsWith("test.")) { // specific case for swt...
+ shortName = shortName.substring(5);
+ }
+
+ // Remove qualification from test name
+ StringTokenizer tokenizer = new StringTokenizer(shortName, " :,", true);
+ StringBuffer buffer = new StringBuffer(tokenizer.nextToken());
+ while (tokenizer.hasMoreTokens()) {
+ String token = tokenizer.nextToken();
+ char fc = token.charAt(0);
+ while (fc == ' ' || fc == ',' || fc == ':') {
+ buffer.append(token); // add the separator
+ token = tokenizer.nextToken();
+ fc = token.charAt(0);
+ }
+ int last = token .lastIndexOf('.');
+ if (last >= 3) {
+ int first = token .indexOf('.');
+ if (first == last) {
+ buffer.append(token);
+ } else {
+// buffer.append(token.substring(0, first));
+// buffer.append("...");
+ buffer.append(token.substring(last+1));
+ }
+ } else {
+ buffer.append(token);
+ }
+ }
+ if (max > 0 && buffer.length() > max) {
+ return "*"+buffer.substring(0, max);
+ }
+ return buffer.toString();
+ }
+}
\ No newline at end of file
Index: src/org/eclipse/test/performance/ui/ScenarioStatusTable.java
===================================================================
RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/ScenarioStatusTable.java,v
retrieving revision 1.28
diff -u -r1.28 ScenarioStatusTable.java
--- src/org/eclipse/test/performance/ui/ScenarioStatusTable.java 15 May 2007 19:23:10 -0000 1.28
+++ src/org/eclipse/test/performance/ui/ScenarioStatusTable.java 1 Jun 2007 16:28:56 -0000
@@ -10,12 +10,19 @@
*******************************************************************************/
package org.eclipse.test.performance.ui;
+import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Hashtable;
+import java.util.StringTokenizer;
+import org.eclipse.test.internal.performance.PerformanceTestPlugin;
+import org.eclipse.test.internal.performance.data.Dim;
import org.eclipse.test.internal.performance.db.DB;
import org.eclipse.test.internal.performance.db.Scenario;
+import org.eclipse.test.internal.performance.db.SummaryEntry;
+import org.eclipse.test.internal.performance.db.TimeSeries;
import org.eclipse.test.internal.performance.db.Variations;
+import org.eclipse.test.performance.Dimension;
public class ScenarioStatusTable {
@@ -24,18 +31,23 @@
private String scenarioPattern;
private ArrayList configNames=new ArrayList();
private Hashtable scenarioComments;
+ private SummaryEntry[] fingerprintEntries;
private final String baseline;
private class ScenarioStatus{
Hashtable statusMap;
- String name;
+ String name, shortName;
Hashtable configStatus;
+ Hashtable resultsMap;
boolean hasSlowDownExplanation=false;
+ boolean fingerprint = false;
+ boolean hasBaseline = true;
public ScenarioStatus(String scenarioName){
- name=scenarioName;
- statusMap=new Hashtable();
- configStatus=new Hashtable();
+ name = scenarioName;
+ statusMap = new Hashtable();
+ configStatus = new Hashtable();
+ resultsMap = new Hashtable();
}
}
@@ -48,20 +60,20 @@
* @param scenarioPattern
* @param configDescriptors
*/
- public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, String baseline){
+ public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, SummaryEntry[] fpSummaries, String baseline){
configMaps=configDescriptors;
this.variations=variations;
this.scenarioPattern=scenarioPattern;
this.scenarioComments=scenarioComments;
this.baseline= baseline;
+ this.fingerprintEntries = fpSummaries;
}
-
+
/**
- * Returns HTML representation of scenario status table.
+ * Prints the HTML representation of scenario status table into the given stream.
*/
- public String toString() {
+ public void print(PrintStream stream, boolean filter) {
String OS="config";
- String htmlTable="";
Scenario[] scenarios= DB.queryScenarios(variations, scenarioPattern, OS, null);
if (scenarios != null && scenarios.length > 0) {
@@ -70,25 +82,40 @@
for (int i= 0; i < scenarios.length; i++) {
Scenario scenario= scenarios[i];
String scenarioName=scenario.getScenarioName();
-// if (!Utils.matchPattern(scenarioName, scenarioPattern)) continue;
+ if (filter && !Utils.matchPattern(scenarioName, scenarioPattern)) continue;
// returns the config names. Making assumption that indices in
// the configs array map to the indices of the failure messages.
String[] configs=scenario.getTimeSeriesLabels();
String[] failureMessages= scenario.getFailureMessages();
ScenarioStatus scenarioStatus=new ScenarioStatus(scenarioName);
- scenarioStatusList.add(scenarioStatus);
+ scenarioStatus.fingerprint = Utils.hasSummary(this.fingerprintEntries, scenarioName);
String scenarioComment= (String)scenarioComments.get(scenarioName);
if (scenarioComment != null)
scenarioStatus.hasSlowDownExplanation= true;
-
- for (int j=0;j(vs. ");
+ shortName.append(timeSeriesLabels[0]);
+ shortName.append(")");
+ scenarioStatus.shortName = shortName.toString();
+ }
+ }
+
+ // Store scenario status
+ if (!scenarioStatusList.contains(scenarioStatus)) {
+ scenarioStatusList.add(scenarioStatus);
+ }
}
}
String label=null;
- htmlTable=htmlTable.concat("
Scenario Status
\n" +
- "The scenario status table shows all scenarios tests result for all performance test machines. It gives a complete but compact view of performance result for the component.
\n" +
- "For each test (ie. in each cell of this table), following information are displayed:\n" +
- "\n" +
- "- an icon showing whether the test fails or passes and whether it's reliable or not.
\n" +
- "The legend for this icon is:\n" +
- "\n" +
- "- Green (): mark a successful result, which means this test has neither significant performance regression nor significant standard error
\n" +
- "- Red (): mark a failing result, which means this test shows a significant performance regression (more than 10%)
\n" +
- "- Gray (): mark a failing result (see above) with a comment explaining this degradation.
\n" +
- "- Yellow ( or ): mark a failing or successful result with a significant standard error (more than "+Utils.STANDARD_ERROR_THRESHOLD_STRING+")
\n" +
- "- \"n/a\": mark a test for with no performance results
\n" +
- "
\n" +
- "- the value of the deviation from the baseline as a percentage (ie. formula is:
(build_test_time - baseline_test_time) / baseline_test_time
) \n" +
- "- the value of the standard error of this deviation as a percentage (ie. formula is:
sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time
)
\n" +
- "Note that errors equal to 0 are not shown (tests which have only one iteration). \n" +
- "
" +
- "For failing tests, value of deviation with its standard error is added at the beginning of the error message you can see flying over the corresponding image.
\n" +
- "Follow the link on test box corresponding image for detailed results.
" +
- "
\n");
+ stream.println("
Scenario Status
");
+ stream.println("The following table gives a complete but compact view of performance results for the component.
");
+ stream.println("Each line of the table shows the results for one scenario on all machines.
");
+ stream.println("The name of the scenario is bolded when its results are also displayed in the fingerprints
");
+ stream.println("and starts with an '*' when the scenario has no results in the last baseline run.
");
+ stream.println("Here are information displayed for each test (ie. in each cell):");
+ stream.println("");
+ stream.println("- an icon showing whether the test fails or passes and whether it's reliable or not.
");
+ stream.println("The legend for this icon is:");
+ stream.println("");
+ stream.print("- Green (): mark a successful result, which means this test has neither significant performance regression nor significant standard error
");
+ stream.print("- Red (): mark a failing result, which means this test shows a significant performance regression (more than 10%)
");
+ stream.print("- Gray (): mark a failing result (see above) with a comment explaining this degradation.
");
+ stream.print("- Yellow ( or ): mark a failing or successful result with a significant standard error (more than ");
+ stream.print(Utils.STANDARD_ERROR_THRESHOLD_STRING);
+ stream.println(")
");
+ stream.print("- Black (): mark an undefined result, which means that deviation on this test is not a number (
NaN
) or is infinite (happens when the reference value is equals to 0!) ");
+ stream.println("- \"n/a\": mark a test for with no performance results
");
+ stream.println("
");
+ stream.println("- the value of the deviation from the baseline as a percentage (ie. formula is:
(build_test_time - baseline_test_time) / baseline_test_time
) ");
+ stream.println("- the value of the standard error of this deviation as a percentage (ie. formula is:
sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time
)
");
+ stream.println("When test only has one measure, the standard error cannot be computed and is replaced with a '[n/a]'. ");
+ stream.println("
");
+ stream.println("Hints:");
+ stream.println("- fly over image of failing tests to see the complete error message
");
+ stream.println("- to look at the complete and detailed test results, click on its image
");
+ stream.println("
");
+ stream.println();
+ stream.println("");
+ stream.println("");
+ stream.print("All ");
+ stream.print(scenarios.length);
+ stream.println(" scenarios | ");
- htmlTable=htmlTable.concat("All "+scenarios.length+" scenarios | \n");
for (int i= 0; i < configNames.size(); i++){
- label=configNames.get(i).toString();
- String columnTitle=label;
+ label = configNames.get(i).toString();
+ String columnTitle = label;
if (configMaps!=null) {
Utils.ConfigDescriptor configDescriptor= (Utils.ConfigDescriptor)configMaps.get(label);
- if (configDescriptor != null)
- columnTitle=configDescriptor.description;
+ if (configDescriptor != null) {
+ int idx = configDescriptor.description.indexOf('(');
+ if (idx < 0) {
+ columnTitle = configDescriptor.description;
+ } else {
+ // first line
+ StringTokenizer tokenizer = new StringTokenizer(configDescriptor.description.substring(0, idx).trim(), " ");
+ StringBuffer buffer = new StringBuffer(tokenizer.nextToken());
+ while (tokenizer.hasMoreTokens()) {
+ buffer.append(" ");
+ buffer.append(tokenizer.nextToken());
+ }
+ buffer.append(' ');
+ // second line
+ tokenizer = new StringTokenizer(configDescriptor.description.substring(idx).trim(), " ");
+ buffer.append(tokenizer.nextToken());
+ while (tokenizer.hasMoreTokens()) {
+ buffer.append(" ");
+ buffer.append(tokenizer.nextToken());
+ }
+ columnTitle = buffer.toString();
+ }
+ }
}
- htmlTable=htmlTable.concat(""+columnTitle +" | ");
+ stream.print("");
+ stream.print(columnTitle);
+ stream.println("");
}
-
- htmlTable=htmlTable.concat(" |
\n");
// counter for js class Id's
int jsIdCount=0;
@@ -147,61 +240,63 @@
ScenarioStatus status=(ScenarioStatus)scenarioStatusList.get(j);
- htmlTable=htmlTable.concat(""+status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1)+" | ");
+ stream.println("
");
+ stream.print("");
+ if (status.fingerprint) stream.print("");
+ if (!status.hasBaseline) stream.print("*");
+// stream.print(status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1));
+ stream.print(status.shortName);
+ if (!status.hasBaseline) stream.print("");
+ if (status.fingerprint) stream.print("");
+ stream.println();
for (int i=0;i\n");
+ stream.print("href=\"");
+ stream.print(aUrl);
+ stream.print('/');
+ stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
+ stream.println(".html\">");
+ stream.print("");
} else {
// create message with tooltip text including deviation with error plus failure message
jsIdCount+=1;
- html.append("class=\"tooltipSource\" onMouseover=\"show_element('toolTip");
- html.append(jsIdCount);
- html.append("')\" onMouseout=\"hide_element('toolTip");
- html.append(jsIdCount);
- html.append("')\" \nhref=\"");
- html.append(aUrl);
- html.append('/');
- html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
- html.append(".html\">\n\n");
- html.append(message);
- html.append("");
+ stream.print("class=\"tooltipSource\" onMouseover=\"show_element('toolTip");
+ stream.print(jsIdCount);
+ stream.print("')\" onMouseout=\"hide_element('toolTip");
+ stream.print(jsIdCount);
+ stream.print("')\" \nhref=\"");
+ stream.print(aUrl);
+ stream.print('/');
+ stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
+ stream.println(".html\">");
+ stream.print("");
+ stream.print("");
+ stream.print(message);
+ stream.println("");
}
- html.append(Utils.failureMessage(resultStats, false));
- html.append(" | ");
- htmlTable=htmlTable.concat(html.toString());
+ String result = (String) status.resultsMap.get(configName);
+ stream.println(result);
}else{
- htmlTable=htmlTable.concat("n/a | ");
+ stream.println(" n/a");
}
}
+ stream.flush();
}
-
- htmlTable=htmlTable.concat(" |
\n");
+ stream.println("
");
}
- return htmlTable;
- }
+ }
}
Index: src/org/eclipse/test/performance/ui/FingerPrint.java
===================================================================
RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/FingerPrint.java,v
retrieving revision 1.20
diff -u -r1.20 FingerPrint.java
--- src/org/eclipse/test/performance/ui/FingerPrint.java 27 Apr 2007 19:07:44 -0000 1.20
+++ src/org/eclipse/test/performance/ui/FingerPrint.java 1 Jun 2007 16:28:55 -0000
@@ -19,17 +19,11 @@
import java.util.Hashtable;
import org.eclipse.swt.SWT;
-import org.eclipse.swt.events.PaintEvent;
-import org.eclipse.swt.events.PaintListener;
import org.eclipse.swt.graphics.GC;
import org.eclipse.swt.graphics.Image;
import org.eclipse.swt.graphics.ImageData;
import org.eclipse.swt.graphics.ImageLoader;
-import org.eclipse.swt.graphics.Point;
-import org.eclipse.swt.layout.FillLayout;
-import org.eclipse.swt.widgets.Canvas;
import org.eclipse.swt.widgets.Display;
-import org.eclipse.swt.widgets.Shell;
import org.eclipse.test.internal.performance.PerformanceTestPlugin;
import org.eclipse.test.internal.performance.data.Dim;
import org.eclipse.test.internal.performance.db.DB;
@@ -37,9 +31,6 @@
import org.eclipse.test.internal.performance.db.SummaryEntry;
import org.eclipse.test.internal.performance.db.TimeSeries;
import org.eclipse.test.internal.performance.db.Variations;
-import org.eclipse.test.internal.performance.eval.StatisticsUtil;
-import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
-import org.eclipse.test.performance.Dimension;
import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;
@@ -74,17 +65,16 @@
if (component==null){
entries= DB.querySummaries(variations,null);
this.component="";
- }
- else
+ } else {
entries=DB.querySummaries(variations,component+'%');
- run(entries);
+ }
+ run();
}
/**
* Creates the fingerprint gif, image map and scenario status table for the component.
- * @param entries - the result of a database query for summaries for a specified variation.
*/
- public void run(Object[] entries) {
+ public void run() {
new File(outputDirectory).mkdirs();
String referenceName=referenceBuildId;
String currentName=currentBuildId;
@@ -101,12 +91,12 @@
if (entries != null) {
for (int i= 0; i < entries.length; i++) {
- SummaryEntry se= (SummaryEntry)entries[i];
- if (se.comment==null)
- add(bar, se.shortName, new Dim[] { se.dimension }, se.scenarioName);
+ SummaryEntry summary = entries[i];
+ if (summary.comment==null)
+ add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName);
else{
- setComment(se.scenarioName, se.comment);
- add(bar, se.shortName, new Dim[] { se.dimension }, se.scenarioName,se.comment);
+ setComment(summary.scenarioName, summary.comment);
+ add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName, summary.comment);
}
}
}
@@ -115,7 +105,7 @@
if (component=="")
outName= "FP_"+referenceName + '_' + currentBuildId+"."+configDescriptor.name;
- save(bar, outputDirectory + '/' + outName);
+ save(outputDirectory + '/' + outName);
//show(bar);
@@ -127,11 +117,11 @@
scenarioComments.put(scenario,comment);
}
- private void add(BarGraph bar, String name, Dim[] dims, String scenarioName) {
- add (bar,name,dims,scenarioName,null);
+ private void add(String name, Dim[] dims, String scenarioName) {
+ add (name,dims,scenarioName,null);
}
- private void add(BarGraph bar, String name, Dim[] dims, String scenarioName,String comment) {
+ private void add(String name, Dim[] dims, String scenarioName, String comment) {
String refData= "";
Scenario scenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, referenceBuildId, currentBuildId, dims);
String[] timeSeriesLabels= scenario.getTimeSeriesLabels();
@@ -153,7 +143,8 @@
double percent= 0.0;
boolean hasConfidentResult= true;
if (l > 1) {
- hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name);
+// hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name);
+ hasConfidentResult= Utils.hasConfidentResult(timeSeries);
/*if (!rejectNullHypothesis) {
NumberFormat percentFormatter= NumberFormat.getPercentInstance();
String statisticsComment= "There is not enough evidence to reject the null hypothesis at the " + percentFormatter.format(percentile.inside()) + "level";
@@ -178,7 +169,7 @@
}
- private void save(BarGraph bar, String output) {
+ private void save(String output) {
//if (bar.getFItems().size()==0)
//return;
@@ -217,7 +208,7 @@
/*
* Displays bar graph in window
- */
+ *
private void show(final BarGraph bar) {
Display display= new Display();
@@ -246,4 +237,23 @@
public String getOutName() {
return outName;
}
+ */
+
+ /**
+ * @return - an html representation of the fingerprint.
+ */
+ public String getImageMap() {
+ String componentDescription = this.configDescriptor.description;
+ String areas = this.bar.getAreas();
+ if (areas == null)
+ areas = "";
+ String output = "";
+ if (new File(this.outputDirectory, this.outName + ".gif").exists()) {
+ output = "" + componentDescription + "
";
+ output = output.concat("" + "\n");
+ } else {
+ output = output.concat("
There is no fingerprint for " + componentDescription + "
\n");
+ }
+ return output;
+ }
}
Index: src/org/eclipse/test/performance/ui/ScenarioResults.java
===================================================================
RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/ScenarioResults.java,v
retrieving revision 1.32
diff -u -r1.32 ScenarioResults.java
--- src/org/eclipse/test/performance/ui/ScenarioResults.java 15 May 2007 19:23:10 -0000 1.32
+++ src/org/eclipse/test/performance/ui/ScenarioResults.java 1 Jun 2007 16:28:56 -0000
@@ -15,7 +15,6 @@
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
-import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Hashtable;
@@ -48,7 +47,7 @@
*
* @param scenarios -
* the array of Scenario objects for which to generate results.
- * @param reference -
+ * @param baseline -
* the reference build ID
* @param current -
* the current buildID
Index: src/org/eclipse/test/performance/ui/Main.java
===================================================================
RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/Main.java,v
retrieving revision 1.40
diff -u -r1.40 Main.java
--- src/org/eclipse/test/performance/ui/Main.java 15 May 2007 19:23:10 -0000 1.40
+++ src/org/eclipse/test/performance/ui/Main.java 1 Jun 2007 16:28:55 -0000
@@ -25,6 +25,7 @@
import org.eclipse.core.runtime.Platform;
import org.eclipse.test.internal.performance.db.Scenario;
+import org.eclipse.test.internal.performance.db.SummaryEntry;
import org.eclipse.test.internal.performance.db.Variations;
import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;
@@ -48,14 +49,21 @@
private Hashtable fingerPrints = new Hashtable();
private Hashtable scenarioComments=new Hashtable();
private Hashtable rawDataTables=new Hashtable();
+ private boolean local = false;
public Object run(Object args) throws Exception {
parse(args);
- Enumeration configIds=configDescriptors.keys();
-
- while (configIds.hasMoreElements()){
- generate((ConfigDescriptor)configDescriptors.get(configIds.nextElement()));
+ if (this.local) {
+ int length = configNames.length;
+ for (int i=0; i");
Hashtable fps = (Hashtable) fingerPrints.get(component);
- Enumeration configs = fps.keys();
int baselineUnderScoreIndex=baseline.indexOf("_");
int currentUnderScoreIndex=currentBuildId.indexOf("_");
String baselineName=(baselineUnderScoreIndex!=-1)?baseline.substring(0, baseline.indexOf("_")):baseline;
String currentName=(currentUnderScoreIndex!=-1)?currentBuildId.substring(0, currentBuildId.indexOf("_")):currentBuildId;
- String title = "Performance of " + component + ": "
- + currentName + " relative to "
- + baselineName
- + "
";
- if (component.equals("global"))
- title = "Performance of " + currentName
- + " relative to "
- + baselineName
- + "
";
- os.println(title);
+ boolean isGlobal = component.equals("global");
+ StringBuffer title = new StringBuffer("Performance of ");
+ if (!isGlobal) {
+ title.append(component);
+ title.append(": ");
+ }
+ title.append(currentName);
+ title.append(" relative to ");
+ title.append(baselineName);
+ title.append( "
");
+ os.println(title.toString());
//print the html representation of fingerprint for each config
+ Enumeration configs = fps.keys();
+ SummaryEntry[] fpSummaries = null;
while (configs.hasMoreElements()) {
String config = configs.nextElement().toString();
- FingerPrint fp = (FingerPrint) fps.get(config);
- os.println(Utils.getImageMap(fp));
- }
- if (component != "") {
- //print the component scenario status table beneath the fingerprint
+ FingerPrint fingerPrint = (FingerPrint) fps.get(config);
+ os.println(fingerPrint.getImageMap());
+ if (fpSummaries == null) {
+ fpSummaries = fingerPrint.entries;
+ }
+ }
+ if (isGlobal) {
+ if (this.local) {
+ os.println("");
+ os.println(" ");
+ os.println("Detailed performance data grouped by scenario prefix |
");
+ os.println("org.eclipse.ant*
");
+ os.println("org.eclipse.compare*
");
+ os.println("org.eclipse.core*
");
+ os.println("org.eclipse.help*
");
+ os.println("org.eclipse.jdt.core*
");
+ os.println("org.eclipse.jdt.debug*
");
+ os.println("org.eclipse.jdt.text*
");
+ os.println("org.eclipse.jdt.ui*
");
+ os.println("org.eclipse.jface*
");
+ os.println("org.eclipse.osgi*
");
+ os.println("org.eclipse.pde.ui*
");
+ os.println("org.eclipse.swt*
");
+ os.println("org.eclipse.team*
");
+ os.println("org.eclipse.ua*
");
+ os.println("org.eclipse.ui*
");
+ }
+ } else if (component.length() > 0) {
+ // print the component scenario status table beneath the fingerprint
variations.put("config", "%");
- ScenarioStatusTable sst = new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, baseline);
-// ScenarioStatusTable sst = new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, baseline);
- os.println(sst.toString());
+ boolean filter = this.local && this.scenarioFilter != null; // use scenario filter to minimize DB requests while testing...
+ ScenarioStatusTable sst = filter
+ ? new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, fpSummaries, baseline)
+ : new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, fpSummaries, baseline);
+ sst.print(os, filter);
}
os.println(Utils.HTML_CLOSE);
@@ -292,8 +328,7 @@
System.out.println("Missing value for -config.properties parameter");
printUsage();
}
- configDescriptors = Utils
- .getConfigDescriptors(configProperties);
+ configDescriptors = Utils.getConfigDescriptors(configProperties);
i++;
continue;
}
@@ -318,6 +353,11 @@
i++;
continue;
}
+ if (arg.equals("-local")) {
+ this.local = true;
+ i++;
+ continue;
+ }
i++;
}
Index: src/org/eclipse/test/performance/ui/Messages.java
===================================================================
RCS file: src/org/eclipse/test/performance/ui/Messages.java
diff -N src/org/eclipse/test/performance/ui/Messages.java
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ src/org/eclipse/test/performance/ui/Messages.java 1 Jan 1970 00:00:00 -0000
@@ -0,0 +1,75 @@
+/*******************************************************************************
+ * Copyright (c) 2000, 2007 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * IBM Corporation - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.test.performance.ui;
+
+import java.text.MessageFormat;
+
+import org.eclipse.osgi.util.NLS;
+
+public class Messages extends NLS {
+
+ private static final String BUNDLE_NAME = "org.eclipse.test.performance.ui.messages";//$NON-NLS-1$
+
+ private Messages() {
+ // Do not instantiate
+ }
+
+ public static String standardError;
+
+ static {
+ NLS.initializeMessages(BUNDLE_NAME, Messages.class);
+ }
+
+ /**
+ * Bind the given message's substitution locations with the given string values.
+ *
+ * @param message the message to be manipulated
+ * @return the manipulated String
+ */
+ public static String bind(String message) {
+ return bind(message, null);
+ }
+
+ /**
+ * Bind the given message's substitution locations with the given string values.
+ *
+ * @param message the message to be manipulated
+ * @param binding the object to be inserted into the message
+ * @return the manipulated String
+ */
+ public static String bind(String message, Object binding) {
+ return bind(message, new Object[] {binding});
+ }
+
+ /**
+ * Bind the given message's substitution locations with the given string values.
+ *
+ * @param message the message to be manipulated
+ * @param binding1 An object to be inserted into the message
+ * @param binding2 A second object to be inserted into the message
+ * @return the manipulated String
+ */
+ public static String bind(String message, Object binding1, Object binding2) {
+ return bind(message, new Object[] {binding1, binding2});
+ }
+
+ /**
+ * Bind the given message's substitution locations with the given string values.
+ *
+ * @param message the message to be manipulated
+ * @param bindings An array of objects to be inserted into the message
+ * @return the manipulated String
+ */
+ public static String bind(String message, Object[] bindings) {
+ return MessageFormat.format(message, bindings);
+ }
+
+}
Index: src/org/eclipse/test/performance/ui/messages.properties
===================================================================
RCS file: src/org/eclipse/test/performance/ui/messages.properties
diff -N src/org/eclipse/test/performance/ui/messages.properties
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ src/org/eclipse/test/performance/ui/messages.properties 1 Jan 1970 00:00:00 -0000
@@ -0,0 +1,14 @@
+###############################################################################
+# Copyright (c) 2000, 2007 IBM Corporation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+#
+# Contributors:
+# IBM Corporation - initial API and implementation
+###############################################################################
+
+### Performance Tests Messages.
+
+standardError = Standard error on this test is {0} (should be less than {1} to become reliable!)