### Eclipse Workspace Patch 1.0 #P org.eclipse.test.performance.ui Index: src/org/eclipse/test/performance/ui/Main.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/Main.java,v retrieving revision 1.43 diff -u -r1.43 Main.java --- src/org/eclipse/test/performance/ui/Main.java 5 Jun 2007 13:24:06 -0000 1.43 +++ src/org/eclipse/test/performance/ui/Main.java 31 Aug 2007 15:14:57 -0000 @@ -10,432 +10,724 @@ *******************************************************************************/ package org.eclipse.test.performance.ui; -import java.io.File; -import java.io.FileOutputStream; -import java.io.PrintStream; +import java.io.*; import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.Hashtable; +import java.util.*; +import org.eclipse.core.runtime.FileLocator; +import org.eclipse.equinox.app.IApplication; +import org.eclipse.equinox.app.IApplicationContext; +import org.eclipse.test.internal.performance.PerformanceTestPlugin; +import org.eclipse.test.internal.performance.results.AbstractResults; +import org.eclipse.test.internal.performance.results.ConfigResults; +import org.eclipse.test.internal.performance.results.DB_Results; +import org.eclipse.test.internal.performance.results.PerformanceResults; +import org.eclipse.test.internal.performance.results.ScenarioResults; import org.osgi.framework.Bundle; -import org.eclipse.core.runtime.IPlatformRunnable; -import org.eclipse.core.runtime.Platform; - -import org.eclipse.test.internal.performance.db.Scenario; -import org.eclipse.test.internal.performance.db.SummaryEntry; -import org.eclipse.test.internal.performance.db.Variations; -import org.eclipse.test.performance.ui.Utils.ConfigDescriptor; - -public class Main implements IPlatformRunnable{ - - private String baseline; - private String baselinePrefix=null; - private String output; - private String[] configNames; - private Hashtable configDescriptors; - private String currentBuildId; - private ArrayList currentBuildStreamIdPrefixes; - private ArrayList pointsOfInterest; - private Variations variations; - private Scenario[] scenarios; - private String jvm; - private String scenarioFilter; - private boolean genFingerPrints = false; - private boolean genScenarioSummaries =false; - private boolean genAll = true; - private Hashtable fingerPrints = new Hashtable(); - private Hashtable scenarioComments=new Hashtable(); - private Hashtable rawDataTables=new Hashtable(); - private boolean local = false; - - public Object run(Object args) throws Exception { - parse(args); - - try { - if (this.local) { - int length = configNames.length; - for (int i=0; i"+ - ""); - os.println(Utils.HTML_DEFAULT_CSS); - os.println(""); - Hashtable fps = (Hashtable) fingerPrints.get(component); - - int baselineUnderScoreIndex=baseline.indexOf("_"); - int currentUnderScoreIndex=currentBuildId.indexOf("_"); - - String baselineName=(baselineUnderScoreIndex!=-1)?baseline.substring(0, baseline.indexOf("_")):baseline; - String currentName=(currentUnderScoreIndex!=-1)?currentBuildId.substring(0, currentBuildId.indexOf("_")):currentBuildId; - boolean isGlobal = component.equals("global"); - StringBuffer title = new StringBuffer("

Performance of "); - if (!isGlobal) { - title.append(component); - title.append(": "); - } - title.append(currentName); - title.append(" relative to "); - title.append(baselineName); - title.append( "

"); - os.println(title.toString()); - - //print the html representation of fingerprint for each config - Enumeration configs = fps.keys(); - SummaryEntry[] fpSummaries = null; - while (configs.hasMoreElements()) { - String config = configs.nextElement().toString(); - FingerPrint fingerPrint = (FingerPrint) fps.get(config); - os.println(fingerPrint.getImageMap()); - if (fpSummaries == null) { - fpSummaries = fingerPrint.entries; - } - } - if (isGlobal) { - if (this.local) { - os.println(""); - os.println("
"); - os.println("Detailed performance data grouped by scenario prefix
"); - os.println("org.eclipse.ant*
"); - os.println("org.eclipse.compare*
"); - os.println("org.eclipse.core*
"); - os.println("org.eclipse.help*
"); - os.println("org.eclipse.jdt.core*
"); - os.println("org.eclipse.jdt.debug*
"); - os.println("org.eclipse.jdt.text*
"); - os.println("org.eclipse.jdt.ui*
"); - os.println("org.eclipse.jface*
"); - os.println("org.eclipse.osgi*
"); - os.println("org.eclipse.pde.ui*
"); - os.println("org.eclipse.swt*
"); - os.println("org.eclipse.team*
"); - os.println("org.eclipse.ua*
"); - os.println("org.eclipse.ui*



"); - } - } else if (component.length() > 0) { - // print the component scenario status table beneath the fingerprint - variations.put("config", "%"); - boolean filter = this.local && this.scenarioFilter != null; // use scenario filter to minimize DB requests while testing... - ScenarioStatusTable sst = filter - ? new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, fpSummaries, baseline) - : new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, fpSummaries, baseline); - sst.print(os, filter); - } +/** + * Main class to generate performance results of all scenarii matching a given pattern + * in one HTML page per component. + * + * @see #printUsage() method to see a detailed parameters usage + */ +public class Main implements IApplication { + +/** + * Prefix of baseline builds displayed in data graphs. + * This field is set using -baselinePrefix argument. + *

+ * Example: + * 

-baseline.prefix 3.2_200606291905
+ * + * @see #currentBuildPrefixes + */ +private String baselinePrefix = null; + +/** + * Root directory where all files are generated. + * This field is set using -output argument. + *

+ * Example: + * 

-output /releng/results/I20070615-1200/performance
+ */ +private File outputDir; + +/** + * Arrays of 2 strings which contains config information: name and description. + * This field is set using -config and/or -config.properties arguments. + *

+ * Example: + * 

+ * 	-config eclipseperflnx3_R3.3,eclipseperfwin2_R3.3,eclipseperflnx2_R3.3,eclipseperfwin1_R3.3,eclipseperflnx1_R3.3
+ * 	-config.properties
+ * 		"eclipseperfwin1_R3.3,Win XP Sun 1.4.2_08 (2 GHz 512 MB);
+ * 		eclipseperflnx1_R3.3,RHEL 3.0 Sun 1.4.2_08 (2 GHz 512 MB);
+ * 		eclipseperfwin2_R3.3,Win XP Sun 1.4.2_08 (3 GHz 2 GB);
+ * 		eclipseperflnx2_R3.3,RHEL 3.0 Sun 1.4.2_08 (3 GHz 2 GB);
+ * 		eclipseperflnx3_R3.3,RHEL 4.0 Sun 1.4.2_08 (3 GHz 2.5 GB)"
+ *
+ * Note that: + * + */ +private String[][] configDescriptors; + +/** + * Scenario pattern used to generate performance results. + * This field is set using -scenarioPattern argument. + *

+ * Note that this pattern uses SQL conventions, not RegEx ones, + * which means that '%' is used to match several consecutive characters + * and '_' to match a single character. + *

+ * Example: + * 

-scenario.pattern org.eclipse.%.test
+ */ +private String scenarioPattern; + +/** + * A list of prefixes for builds displayed in data graphs. + * This field is set using -currentPrefix argument. + *

+ * Example: + * 

-current.prefix N, I
+ * + * @see #baselinePrefix + */ +private List currentBuildPrefixes; + +/** + * A list of prefixes of builds to highlight in displayed data graphs. + * This field is set using -highlight and/or -highlight.latest arguments. + *

+ * Example: + * 

-higlight 3_2
+ */ +private List pointsOfInterest; + +/** + * Tells whether only fingerprints has to be generated. + * This field is set to true if -fingerprints argument is specified. + *

+ * Default is false which means that scenario data + * will also be generated. + * + * @see #genData + * @see #genAll + */ +private boolean genFingerPrints = false; + +/** + * Tells whether only fingerprints has to be generated. + * This field is set to true if -data argument is specified. + *

+ * Default is false which means that fingerprints + * will also be generated. + * + * @see #genFingerPrints + * @see #genAll + */ +private boolean genData = false; + +/** + * Tells whether only fingerprints has to be generated. + * This field is set to false + * if -fingerprints or -data argument is specified. + *

+ * Default is true which means that scenario data + * will also be generated. + * + * @see #genData + * @see #genFingerPrints + */ +private boolean genAll = true; + +/** + * Tells whether information should be displayed in the console while generating. + * This field is set to true if -print argument is specified. + *

+ * Default is false which means that nothing is print during the generation. + */ +private boolean print = false; + +/* + * Parse the command arguments and create corresponding performance + * results object. + */ +private PerformanceResults parse(Object argsObject) { + StringBuffer buffer = new StringBuffer("Parameters used to generate performance results:\n"); + String[] args = (String[]) argsObject; + int i = 0; + if (args.length == 0) { + printUsage(); + } - os.println(Utils.HTML_CLOSE); - os.close(); + String currentBuildId = null; + String baseline = null; + String jvm = null; + this.configDescriptors = null; + + while (i < args.length) { + String arg = args[i]; + if (!arg.startsWith("-")) { + i++; + continue; + } + if (args.length == i + 1 && i != args.length - 1) { + System.out.println("Missing value for last parameter"); + printUsage(); + } + if (arg.equals("-baseline")) { + baseline = args[i + 1]; + if (baseline.startsWith("-")) { + System.out.println("Missing value for -baseline parameter"); + printUsage(); } - } catch (Exception e) { - // Need to print any unexpected exception otherwise the failure will be completely silent... - e.printStackTrace(); - } - - return null; - } - - - private void generate(Utils.ConfigDescriptor cd) { - //String config=cd.getName(); - String dbloc_property= System.getProperty("eclipse.perf.dbloc"); - if (dbloc_property == null || dbloc_property.equals("")) - System.out.println("WARNING: eclipse.perf.dbloc value set to null"); - scenarios = Utils.getScenarios("%", scenarioFilter, cd.name, jvm); - variations = Utils.getVariations("%", cd.name, jvm); - - //creates and stores fingerprint objects - if (genFingerPrints || genAll) { - System.out.print(cd.name + ": generating fingerprints and scenario status tables..."); - - //global - FingerPrint global = new FingerPrint(null, cd, baseline, currentBuildId, variations, output); - - //store mappings of fingerprints per config for each component - Hashtable t; - if (fingerPrints.get("global") != null) - t = (Hashtable) fingerPrints.get("global"); - else - t = new Hashtable(); - - t.put(cd.name, global); - fingerPrints.put("global", t); - - //get unique component names from scenario names - ArrayList components = Utils.getComponentNames(scenarios); - - //store fingerprints for config for each component - for (int i = 0; i < components.size(); i++) { - String component = components.get(i).toString(); - variations.put("config", cd.name); - FingerPrint componentFp = new FingerPrint(component, cd, baseline, currentBuildId, variations, output); - if (fingerPrints.get(component) != null) - t = (Hashtable) fingerPrints.get(component); - else - t = new Hashtable(); - t.put(cd.name, componentFp); - fingerPrints.put(component, t); - if (componentFp.scenarioComments!=null) - scenarioComments.putAll(componentFp.scenarioComments); - } - System.out.println("done."); + buffer.append(" -baseline = "+baseline+'\n'); + i++; + continue; + } + if (arg.equals("-baseline.prefix")) { + this.baselinePrefix = args[i + 1]; + if (this.baselinePrefix.startsWith("-")) { + System.out.println("Missing value for -baseline.prefix parameter"); + printUsage(); } - - //generates scenario result pages and line graphs - if (genScenarioSummaries || genAll) { - System.out.print(cd.name - + ": generating scenario results..."); - new ScenarioResults(cd,scenarios, baseline,baselinePrefix,currentBuildId,pointsOfInterest,scenarioComments,currentBuildStreamIdPrefixes,rawDataTables,output,variations); - System.out.println("done."); - } - } - - - private void parse(Object argsObject) { - String []args=(String[])argsObject; - int i = 0; - if (args.length == 0) { - printUsage(); + buffer.append(" -baselinePrefix = "+this.baselinePrefix+'\n'); + i++; + continue; } - - while (i < args.length) { - String arg = args[i]; - if (!arg.startsWith("-")){ - i++; - continue; - } - if (args.length==i+1&&i!=args.length-1){ - System.out.println("Missing value for last parameter"); + if (arg.equals("-current.prefix")) { + String idPrefixList = args[i + 1]; + if (idPrefixList.startsWith("-")) { + System.out.println("Missing value for -current.prefix parameter"); printUsage(); } - if (arg.equals("-baseline")) { - baseline = args[i + 1]; - if (baseline.startsWith("-")) { - System.out.println("Missing value for -baseline parameter"); - printUsage(); - } - i++; - continue; + buffer.append(" -current.prefix = "); + String[] ids = idPrefixList.split(","); + this.currentBuildPrefixes = new ArrayList(); + for (int j = 0; j < ids.length; j++) { + this.currentBuildPrefixes.add(ids[j]); + buffer.append(ids[j]); } - if (arg.equals("-baseline.prefix")) { - baselinePrefix = args[i + 1]; - if (baselinePrefix.startsWith("-")) { - System.out.println("Missing value for -baseline.prefix parameter"); - printUsage(); - } - i++; - continue; + buffer.append('\n'); + i++; + continue; + } + if (arg.equals("-highlight") || arg.equals("-highlight.latest")) { + if (args[i + 1].startsWith("-")) { + System.out.println("Missing value for -highlight parameter"); + printUsage(); } - if (arg.equals("-current.prefix")) { - String idPrefixList=args[i + 1]; - if (idPrefixList.startsWith("-")) { - System.out.println("Missing value for -current.prefix parameter"); - printUsage(); - } - String []ids=idPrefixList.split(","); - currentBuildStreamIdPrefixes=new ArrayList(); - for (int j=0;j "); + buffer.append(elements[1]); + } + } } - + buffer.append('\n'); i++; + continue; } - if (baseline == null || output == null || configNames == null - || jvm == null - || currentBuildId == null) - printUsage(); - - if (currentBuildStreamIdPrefixes==null){ - currentBuildStreamIdPrefixes=new ArrayList(); - currentBuildStreamIdPrefixes.add("N"); - currentBuildStreamIdPrefixes.add("I"); - } - if (configDescriptors==null){ - configDescriptors=new Hashtable(); - for (int j=0;j"); + stream.println(Utils.HTML_DEFAULT_CSS); + stream.println(""); + + String baselineName = performanceResults.getBaselineName(); + String currentName = performanceResults.getName(); + boolean isGlobal = component.equals("global"); + StringBuffer title = new StringBuffer("

Performance of "); + if (!isGlobal) { + title.append(component); + title.append(": "); + } + title.append(currentName); + title.append(" relative to "); + int index = baselineName.indexOf('_'); + title.append(baselineName.substring(0, index)); + title.append(" ("); + index = baselineName.lastIndexOf('_'); + title.append(baselineName.substring(index+1, baselineName.length())); + title.append(")

"); + stream.println(title.toString()); + + // print the html representation of fingerprint for each config + if (genFingerPrints || genAll) { + FingerPrint fingerprint = new FingerPrint(component, stream, this.outputDir); + try { + fingerprint.print(performanceResults); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + // print scenario status table + if (isGlobal) { + if (!PerformanceTestPlugin.getDBLocation().startsWith("net://")) { + stream.println(""); + stream.println("
"); + stream.println("Detailed performance data grouped by scenario prefix
"); + stream.println("org.eclipse.ant*
"); + stream.println("org.eclipse.compare*
"); + stream.println("org.eclipse.core*
"); + stream.println("org.eclipse.jdt.core*
"); + stream.println("org.eclipse.jdt.debug*
"); + stream.println("org.eclipse.jdt.text*
"); + stream.println("org.eclipse.jdt.ui*
"); + stream.println("org.eclipse.jface*
"); + stream.println("org.eclipse.osgi*
"); + stream.println("org.eclipse.pde.ui*
"); + stream.println("org.eclipse.swt*
"); + stream.println("org.eclipse.team*
"); + stream.println("org.eclipse.ua*
"); + stream.println("org.eclipse.ui*



"); + } + } else if (component.length() > 0) { + // print the component scenario status table beneath the fingerprint + ScenarioStatusTable sst = new ScenarioStatusTable(component, stream); + try { + sst.print(performanceResults); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + stream.println(Utils.HTML_CLOSE); + stream.close(); +} + +/* + * Print summary of coefficient of variation for each scenario of the given pattern + * both for baseline and current builds. + */ +private void printSummary(PerformanceResults performanceResults) { + long start = System.currentTimeMillis(); + if (this.print) System.out.print("Print scenarii variations summary..."); + File outputFile = new File(this.outputDir, "cvsummary.html"); + PrintStream stream = null; + try { + stream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outputFile))); + printSummaryPresentation(stream); + List scenarioNames = DB_Results.getScenariiNames(); + int size = scenarioNames.size(); + printSummaryColumnsTitle(stream, performanceResults); + String[] configs = performanceResults.getConfigNames(true/*sorted*/); + int configsLength = configs.length; + for (int i=0; i"); + for (int j=0; j<2; j++) { + for (int c=0; c"); + stream.print(scenarioName); + stream.println(""); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + stream.println(""); + stream.flush(); + stream.close(); + } + if (this.print) System.out.println("done in "+(System.currentTimeMillis()-start)+"ms"); +} +/* + * Print summary presentation (eg. file start and text presenting the purpose of this file contents).. + */ +private void printSummaryPresentation(PrintStream stream) { + stream.println(Utils.HTML_OPEN); + stream.print(Utils.HTML_DEFAULT_CSS); + stream.println("Summary of Elapsed Process Variation Coefficients"); + stream.println("

Summary of Elapsed Process Variation Coefficients

\n"); + stream.println("

This table provides a bird's eye view of variability in elapsed process times\n"); + stream.print("for baseline and current build stream performance scenarios."); + stream.print(" This summary is provided to facilitate the identification of scenarios that should be examined due to high variability."); + stream.println("The variability for each scenario is expressed as a coefficient\n"); + stream.println("of variation (CV). The CV is calculated by dividing the standard deviation\n"); + stream.println("of the elapse process time over builds by the average elapsed process\n"); + stream.println("time over builds and multiplying by 100.\n"); + stream.println("

High CV values may be indicative of any of the following:

\n"); + stream.println("
  1. an unstable performance test.
    2. \n"); + stream.println("\n"); + stream.println("
  2. performance regressions or improvements at some time in the course of builds.
    3. \n"); + stream.println("\n"); + stream.println("
  3. unstable testing hardware.\n"); + stream.print("
    • consistent higher CV values for one test configuration as compared to others across"); + stream.println(" scenarios may be related to hardward problems.
\n"); + stream.println("

Scenarios are listed in alphabetical order in the far right column. A scenario's\n"); + stream.println("variation coefficients (CVs) are in columns to the left for baseline and current\n"); + stream.println("build streams for each test configuration. Scenarios with CVs > 10% are highlighted\n"); + stream.println("in yellow (10%<CV<20%) and orange(CV>20%).

\n"); + stream.println("

Each CV value links to the scenario's detailed results to allow viewers to\n"); + stream.println("investigate the variability.

\n"); +} + +/* + * Print columns titles of the summary table. + */ +private void printSummaryColumnsTitle(PrintStream stream, PerformanceResults performanceResults) { + String[] configBoxes = performanceResults.getConfigBoxes(true/*sorted*/); + int length = configBoxes.length; + stream.print(""); + stream.print(""); + for (int n=0; n<2; n++) { + for (int c=0; c"); + stream.print(configBoxes[c]); + stream.print(""); + } } + stream.println("\n"); +} +/* + * Print a scenario line in the summary table. + */ +private void printSummaryScenarioLine(int i, String config, ScenarioResults scenarioResults, PrintStream stream) { + ConfigResults configResults = scenarioResults.getConfigResults(config); + if (configResults == null || !configResults.isValid()) { + stream.print(""); + return; + } + String url = config + "/" + scenarioResults.getFileName()+".html"; + double[] stats = null; + int dim_id = AbstractResults.SUPPORTED_DIMS[0].getId(); + if (i==0) { // baseline results + List baselinePrefixes = new ArrayList(); + baselinePrefixes.add(this.baselinePrefix); + stats = configResults.getStatistics(baselinePrefixes, dim_id); + } else { + stats = configResults.getStatistics(this.currentBuildPrefixes, dim_id); + } + double variation = stats[3]; + if (variation > 10 && variation < 20) { + stream.print(""); +} + +/* + * Print usage in case one of the argument of the line was incorrect. + * Note that calling this method ends the program run due to final System.exit() + */ +private void printUsage() { + System.out.println( + "Usage:\n\n" + + "-baseline\n" + + " Build id against which to compare results.\n" + + " Same as value specified for the \"build\" key in the eclipse.perf.config system property.\n\n" + + + "[-baseline.prefix]\n" + + " Optional. Build id prefix used in baseline test builds and reruns. Used to plot baseline historical data.\n" + + " A common prefix used for the value of the \"build\" key in the eclipse.perf.config system property when rerunning baseline tests.\n\n" + + + "-current\n" + + " build id for which to generate results. Compared to build id specified in -baseline parameter above.\n" + + " Same as value specified for the \"build\" key in the eclipse.perf.config system property. \n\n" + + + "[-current.prefix]\n" + + " Optional. Comma separated list of build id prefixes used in current build stream.\n" + + " Used to plot current build stream historical data. Defaults to \"N,I\".\n" + + " Prefixes for values specified for the \"build\" key in the eclipse.perf.config system property. \n\n" + + + "-jvm\n" + + " Value specified in \"jvm\" key in eclipse.perf.config system property for current build.\n\n" + + + "-config\n" + + " Comma separated list of config names for which to generate results.\n" + + " Same as values specified in \"config\" key in eclipse.perf.config system property.\n\n" + + + "-output\n" + + " Path to default output directory.\n\n" + + + "[-config.properties]\n" + + " Optional. Used by scenario status table to provide the following:\n" + + " alternate descriptions of config values to use in columns.\n" + + " The value should be specified in the following format:\n" + + " name1,description1;name2,description2;etc..\n\n" + + + "[-highlight]\n" + + " Optional. Comma-separated list of build Id prefixes used to find most recent matching for each entry.\n" + + " Result used to highlight points in line graphs.\n\n" + + + "[-scenario.pattern]\n" + + " Optional. Scenario prefix pattern to query database. If not specified,\n" + + " default of % used in query.\n\n" + + + "[-fingerprints]\n" + + " Optional. Use to generate fingerprints only.\n\n" + + + "[-data]\n" + + " Optional. Generates table of scenario reference and current data with line graphs.\n\n" + + + "[-print]\n" + + " Optional. Display output in the console while generating.\n" + ); + + System.exit(1); +} + +/** + * Generate the performance results for a specified build regarding to a specific reference. + * This action generates following HTML files: + * + * @see org.eclipse.equinox.app.IApplication#start(org.eclipse.equinox.app.IApplicationContext) + */ +public Object start(IApplicationContext context) throws Exception { + + // Parse arguments and read DB info + PerformanceResults performanceResults = parse(context.getArguments().get("application.args")); + performanceResults.read(this.configDescriptors, this.scenarioPattern); + + // Print whole scenarii summary + printSummary(performanceResults); + + // Copy images and scripts to output dir + Bundle bundle = UiPlugin.getDefault().getBundle(); + URL images = bundle.getEntry("images"); + URL scripts = bundle.getEntry("scripts"); + if (images != null) { + images = FileLocator.resolve(images); + Utils.copyImages(new File(images.getPath()), this.outputDir); + } + if (scripts != null) { + scripts = FileLocator.resolve(scripts); + Utils.copyScripts(new File(scripts.getPath()), this.outputDir); + } + // Print HTML pages and all linked files + if (this.print) { + System.out.println("Print performance results HTML pages:"); + System.out.print(" - all components"); + } + long start = System.currentTimeMillis(); + printComponent(performanceResults, "global"); + Iterator components = performanceResults.getComponents().iterator(); + while (components.hasNext()) { + printComponent(performanceResults, (String) components.next()); + } + if (this.print) System.out.println("done in "+(System.currentTimeMillis()-start)+"ms"); + + // Print the scenarii data + if (genData || genAll) { + start = System.currentTimeMillis(); + if (this.print) System.out.print(" - all scenarii data..."); + ScenarioData data = new ScenarioData(this.baselinePrefix, this.pointsOfInterest, this.currentBuildPrefixes, this.outputDir); + try { + data.print(performanceResults); + } catch (Exception ex) { + ex.printStackTrace(); + } + if (this.print) System.out.println("done in "+(System.currentTimeMillis()-start)+"ms"); + } + return null; +} +/* (non-Javadoc) + * @see org.eclipse.equinox.app.IApplication#stop() + */ +public void stop() { + // Do nothing +} } \ No newline at end of file Index: src/org/eclipse/test/performance/ui/BarGraph.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/BarGraph.java,v retrieving revision 1.9 diff -u -r1.9 BarGraph.java --- src/org/eclipse/test/performance/ui/BarGraph.java 12 Mar 2006 21:17:18 -0000 1.9 +++ src/org/eclipse/test/performance/ui/BarGraph.java 31 Aug 2007 15:14:57 -0000 @@ -40,14 +40,15 @@ private static class BarItem { String title; - double value; + double value, error; String url; String slowdownExpected; boolean significant; - BarItem(String t, double v, String u, String slow, boolean sig) { + BarItem(String t, double[] stats, String u, String slow, boolean sig) { title= t; - value= v; + value= -stats[0] * 100; + error = stats[1] * 100; url= u; slowdownExpected= slow; significant= sig; @@ -62,16 +63,8 @@ fItems= new ArrayList(); } - public void addItem(String name, double value) { - fItems.add(new BarItem(name, value, null, null, true)); - } - - public void addItem(String name, double value, String url) { - fItems.add(new BarItem(name, value, url, null, true)); - } - - public void addItem(String name, double value, String url, String slow, boolean significant) { - fItems.add(new BarItem(name, value, url, slow, significant)); + public void addItem(String name, double[] stats, String url, String slow, boolean significant) { + fItems.add(new BarItem(name, stats, url, slow, significant)); } public int getHeight() { @@ -195,9 +188,10 @@ for (int i= 0; i < bars.length; i++) { BarItem bar= bars[i]; - double delta= bar.value; + double delta = bar.value; + double error = bar.error; double orgDelta= delta; - + boolean clamped= false; if (NO_SCALE) { if (delta > max) { Index: src/org/eclipse/test/performance/ui/TimeLineGraph.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/TimeLineGraph.java,v retrieving revision 1.14 diff -u -r1.14 TimeLineGraph.java --- src/org/eclipse/test/performance/ui/TimeLineGraph.java 6 Aug 2007 09:20:37 -0000 1.14 +++ src/org/eclipse/test/performance/ui/TimeLineGraph.java 31 Aug 2007 15:14:57 -0000 @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2004, 2007 IBM Corporation and others. + * Copyright (c) 2004, 2006 IBM Corporation and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -10,19 +10,13 @@ *******************************************************************************/ package org.eclipse.test.performance.ui; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.Enumeration; -import java.util.Hashtable; -import java.util.List; +import java.util.*; import org.eclipse.swt.graphics.Color; import org.eclipse.swt.graphics.GC; import org.eclipse.swt.graphics.Image; import org.eclipse.swt.graphics.Point; import org.eclipse.swt.graphics.Rectangle; - import org.eclipse.test.internal.performance.data.Dim; public class TimeLineGraph extends LineGraph{ @@ -85,8 +79,8 @@ Comparator comparator=new TimeLineGraphItem.GraphItemComparator(); while (_enum.hasMoreElements()) { - List fItems = (List) _enum.nextElement(); - Object[] fItemsArray=fItems.toArray(); + List items = (List) _enum.nextElement(); + Object[] fItemsArray=items.toArray(); Arrays.sort(fItemsArray,comparator); int lastx = 0; int lasty = 0; @@ -161,12 +155,12 @@ } public void addItem(String groupName,String name, String description, double value, Color col, boolean display, long timestamp,boolean isSpecial,boolean drawBaseline) { - List fItems = (List) fItemGroups.get(groupName); + List items = (List) fItemGroups.get(groupName); if (fItemGroups.get(groupName) == null) { - fItems=new ArrayList(); - fItemGroups.put(groupName, fItems); + items=new ArrayList(); + fItemGroups.put(groupName, items); } - fItems.add(new TimeLineGraphItem(name, description, value, col, display, + items.add(new TimeLineGraphItem(name, description, value, col, display, timestamp,isSpecial,drawBaseline)); } @@ -174,9 +168,9 @@ Enumeration _enum=fItemGroups.elements(); double maxItem= 0; while (_enum.hasMoreElements()) { - List fItems = (List) _enum.nextElement(); - for (int i = 0; i < fItems.size(); i++) { - TimeLineGraphItem graphItem = (TimeLineGraphItem) fItems.get(i); + List items = (List) _enum.nextElement(); + for (int i = 0; i < items.size(); i++) { + TimeLineGraphItem graphItem = (TimeLineGraphItem) items.get(i); if (graphItem.value > maxItem) maxItem = graphItem.value; } @@ -191,9 +185,9 @@ double minItem = getMaxItem(); while (_enum.hasMoreElements()) { - List fItems = (List) _enum.nextElement(); - for (int i = 0; i < fItems.size(); i++) { - TimeLineGraphItem graphItem = (TimeLineGraphItem) fItems.get(i); + List items = (List) _enum.nextElement(); + for (int i = 0; i < items.size(); i++) { + TimeLineGraphItem graphItem = (TimeLineGraphItem) items.get(i); if (graphItem.value < minItem) minItem = graphItem.value; } @@ -209,15 +203,15 @@ TimeLineGraphItem mostRecentItem = null; while (_enum.hasMoreElements()) { - List fItems = (List) _enum.nextElement(); - for (int i = 0; i < fItems.size(); i++) { - if (fItems.size() == 1) - return (TimeLineGraphItem) fItems.get(i); + List items = (List) _enum.nextElement(); + for (int i = 0; i < items.size(); i++) { + if (items.size() == 1) + return (TimeLineGraphItem) items.get(i); else { - TimeLineGraphItem graphItem = (TimeLineGraphItem) fItems.get(i); + TimeLineGraphItem graphItem = (TimeLineGraphItem) items.get(i); if (graphItem.timestamp > mostRecentTimestamp) { mostRecentTimestamp = graphItem.timestamp; - mostRecentItem = (TimeLineGraphItem) fItems.get(i); + mostRecentItem = (TimeLineGraphItem) items.get(i); } } } @@ -238,6 +232,7 @@ int n = mainGroup.size(); int xIncrement=width/n; double max=getMaxItem()*1.2; +// double min=getMinItem()*0.8; for (int i = 0; i < n; i++) { TimeLineGraphItem thisItem = (TimeLineGraphItem) fItemsArray[i]; Index: src/org/eclipse/test/performance/ui/Utils.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/Utils.java,v retrieving revision 1.37 diff -u -r1.37 Utils.java --- src/org/eclipse/test/performance/ui/Utils.java 5 Jun 2007 13:24:06 -0000 1.37 +++ src/org/eclipse/test/performance/ui/Utils.java 31 Aug 2007 15:14:57 -0000 @@ -10,43 +10,25 @@ *******************************************************************************/ package org.eclipse.test.performance.ui; -import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.PrintWriter; import java.text.DecimalFormat; import java.text.NumberFormat; -import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.HashMap; -import java.util.Hashtable; -import java.util.StringTokenizer; -import junit.framework.AssertionFailedError; - -import org.eclipse.swt.SWT; -import org.eclipse.swt.graphics.Color; import org.eclipse.swt.graphics.Image; import org.eclipse.swt.graphics.ImageData; -import org.eclipse.swt.graphics.ImageLoader; import org.eclipse.swt.graphics.PaletteData; import org.eclipse.swt.graphics.RGB; -import org.eclipse.swt.widgets.Display; import org.eclipse.test.internal.performance.PerformanceTestPlugin; -import org.eclipse.test.internal.performance.data.Dim; -import org.eclipse.test.internal.performance.db.DB; -import org.eclipse.test.internal.performance.db.Scenario; -import org.eclipse.test.internal.performance.db.SummaryEntry; -import org.eclipse.test.internal.performance.db.TimeSeries; import org.eclipse.test.internal.performance.db.Variations; -import org.eclipse.test.performance.Dimension; public class Utils { @@ -79,69 +61,6 @@ public final static int OK = 0; public final static int NAN = 0x1; public final static int ERR = 0x2; - public final static int DEV = 0x4; - - /** - * @param dimension - * @return A description of the dimension. - */ - public static String getDimensionDescription(String dimension) { - /* Descriptions of dimensions */ - // Windows and Linux - Hashtable descriptions = new Hashtable(); - descriptions.put("cpu time", "Amount of time the process ran on the CPU."); - - descriptions.put("kernel time", "Amount of time the process ran on the CPU, while the CPU was in kernel mode."); - descriptions.put("used java heap", "Change in the amount of memory allocated for Java objects."); - descriptions.put("working set", "Change in the amount of physical memory used by the process (other data resides in swap space)."); - - // Linux - descriptions.put("data size", "Change in the process' data and stack memory size."); - descriptions.put("hard page faults", "Number of memory pages that were loaded from swap space on disk."); - descriptions.put("library size", "Change in the process' library memory size."); - descriptions.put("soft page faults", - "Number of memory pages that were loaded from memory (i.e., they were not mapped in the process' page table, but already present in memory for some reason)."); - descriptions.put("text size", "Change in the process' code memory size, useful with start-up tests."); - - // Windows - descriptions.put("committed", "Change in the amount of allocated memory (both, in physical memory and swap space, can be preallocated for future use)."); - descriptions.put("elapsed process", "Amount of wall-clock time."); - descriptions.put("gdi objects", "Change in the number of GDI (Graphics Device Interface) objects, can be useful for UI tests (particularly start-up tests)."); - descriptions - .put( - "page faults", - "Number of memory pages that were loaded from swap space on disk or from memory (i.e., in the latter case, they were not mapped in the process' page table, but already present in memory for some reason)."); - descriptions.put("system time", "* no longer measured, same as elapsed time, see PerformanceMonitor *"); - descriptions.put("working set peak", "Increase of the maximum working set size, useful with start-up tests."); - - if (descriptions.get(dimension.toLowerCase()) != null) - return descriptions.get(dimension.toLowerCase()).toString(); - return ""; - } - - /** - * @param timeSeriesLabels - - * an array of build ID's with results for a scenario. - * @param current - - * the current build ID - * @return Build Id's of Nightly builds preceding current. - */ - public static ArrayList lastSevenNightlyBuildNames(String[] timeSeriesLabels, String current) { - int currentIndex = getBuildNameIndex(timeSeriesLabels, current); - ArrayList labels = new ArrayList(); - int j = 6; - - for (int i = timeSeriesLabels.length - 1; i > -1; i--) { - if (j == -1) - break; - String timeSeriesLabel = timeSeriesLabels[i]; - if (timeSeriesLabel.startsWith("N") && i < currentIndex) { - labels.add(timeSeriesLabel); - j--; - } - } - return labels; - } /** * Return <html><head><meta http-equiv="Content-Type" @@ -167,75 +86,6 @@ + ".indexsub { font-size: xx-small;; font-family: Arial, Helvetica, sans-serif; color: #8080FF}\n" + ""; /** - * An utility object which stores a name, description, url and optional - * output directory. - */ - public static class ConfigDescriptor { - String name; - String description; - - /** - * - * @param name - * the value specifed for the key config in the - * eclipse.perf.config system.property key value listings. - * ie. relengbuildwin2 (machine name) - * @param description - * a meaningful description to further describe the config. - * ie. win32-win32-x86 Sun 1.4.2_06 - */ - public ConfigDescriptor(String name, String description) { - this.name = name; - this.description = description; - } - - public String getName() { - return name; - } - - public boolean descriptionMatches(String descr) { - return descr.equals(this.description); - } - } - - /** - * @param configDescriptors - * A semi-colon separated listing of config descriptions.
- * Uses the following format: name,description, - * url,outputdir;name2, description2,url2,output2;etc.. - * @return a mapping of config names to their ConfigDescriptors. - */ - public static Hashtable getConfigDescriptors(String configDescriptors) { - // labelMappings in pairs separated by semi-colon ie. - // relengbuildwin2,win32-win32-x86;trelenggtk,linux-gtk-x86 - StringTokenizer tokenizer = new StringTokenizer(configDescriptors, ";"); - Hashtable configMap = new Hashtable(); - - while (tokenizer.hasMoreTokens()) { - String labelDescriptor = tokenizer.nextToken(); - String[] elements = labelDescriptor.trim().split(","); - ConfigDescriptor descriptor = new ConfigDescriptor(elements[0], elements[1]); - configMap.put(elements[0], descriptor); - } - return configMap; - } - - /** - * Queries database with variation composed of buildIdPattern, config and - * jvm. - * - * @return Array of scenarios. - */ - public static Scenario[] getScenarios(String buildIdPattern, String scenarioPattern, String config, String jvm) { - Dim[] qd = null; - if (scenarioPattern == null) - scenarioPattern = ""; - - Variations variations = getVariations(buildIdPattern, config, jvm); - return DB.queryScenarios(variations, scenarioPattern + "%", PerformanceTestPlugin.BUILD, qd); - } - - /** * Creates a Variations object using build id pattern, config and jvm. * * @param buildIdPattern @@ -252,50 +102,6 @@ } /** - * @param scenarios - * @return list of unique component names derived from prefixes to scenario - * names. - */ - public static ArrayList getComponentNames(Scenario[] scenarios) { - ArrayList componentNames = new ArrayList(); - - for (int i = 0; i < scenarios.length; i++) { - String prefix = null; - Scenario scenario = scenarios[i]; - String scenarioName = scenario.getScenarioName(); - - // use part of scenario name prior to .test to identify component - if (scenarioName.indexOf(".test") != -1) { - prefix = scenarioName.substring(0, scenarioName.indexOf(".test")); - if (!componentNames.contains(prefix)) - componentNames.add(prefix); - } - } - return componentNames; - } - - /* - * @param fp - - * a FingerPrint object - * @return - an html representation of the fingerprint. - * - public static String getImageMap(FingerPrint fp) { - String componentDescription = fp.configDescriptor.description; - String areas = fp.bar.getAreas(); - if (areas == null) - areas = ""; - String output = ""; - if (new File(fp.outputDirectory, fp.getOutName() + ".gif").exists()) { - output = "

" + componentDescription + "

"; - output = output.concat("" + "" + areas + "\n"); - } else { - output = output.concat("

There is no fingerprint for " + componentDescription + "

\n"); - } - return output; - } - */ - - /** * Utility method to copy a file. * * @param src the source file. @@ -334,175 +140,6 @@ } /** - * Returns a LineGraph object representing measurements for a scenario over - * builds. - * - * @param t - - * the scenario object. - * @param dimensionName - - * the name of the measurement for which to generate graph. - * @param baseline - - * the reference build to label. - * @param current - - * the current build to label. - * @param pointsOfInterest - - * an array of buildIds. Points for these are highlighted on - * graph. - * @return a LineGraph object. - */ - public static TimeLineGraph getLineGraph(Scenario t, String dimensionName, String baseline, String baselinePrefix, String current, ArrayList pointsOfInterest,ArrayList currentBuildIdPrefixes) { - Display display = Display.getDefault(); - - Color black = display.getSystemColor(SWT.COLOR_BLACK); - Color yellow = display.getSystemColor(SWT.COLOR_DARK_YELLOW); - Color magenta = display.getSystemColor(SWT.COLOR_MAGENTA); - - String scenarioName = t.getScenarioName(); - Dim[] dimensions = t.getDimensions(); - Dim dim = null; - for (int i = 0; i < dimensions.length; i++) { - if (dimensions[i].getName().equals(dimensionName)) - dim = dimensions[i]; - } - - TimeLineGraph graph = new TimeLineGraph(scenarioName + ": " + dimensionName, dim); - TimeSeries ts = null; - try { - ts = t.getTimeSeries(dim); - int n = ts.getLength(); - - if (n > 0) { - boolean currentFound=false; - for (int j = 0; j < n; j++) { - String buildID = ts.getLabel(j); - int underscoreIndex = buildID.indexOf('_'); - String label = (underscoreIndex != -1 && (buildID.equals(baseline) || buildID.equals(current))) ? buildID.substring(0, underscoreIndex) : buildID; - - double value = ts.getValue(j); - - if (buildID.equals(current)) { - Color color = black; - if (buildID.startsWith("N")) - color = yellow; - - graph.addItem("main", label, dim.getDisplayValue(value), value, color, true, getDateFromBuildID(buildID), true); - continue; - } - if (pointsOfInterest.contains(buildID)&&!currentFound) { - graph.addItem("main", label, dim.getDisplayValue(value), value, black, false, getDateFromBuildID(buildID, false), true); - continue; - } - if (buildID.equals(baseline)) { - boolean drawBaseline = (baselinePrefix != null) ? false : true; - graph.addItem("reference", label, dim.getDisplayValue(value), value, magenta, true, getDateFromBuildID(buildID, true), true, drawBaseline); - continue; - } - if (baselinePrefix != null) { - if (buildID.startsWith(baselinePrefix) && !buildID.equals(baseline) && getDateFromBuildID(buildID, true) <= getDateFromBuildID(baseline, true)) { - graph.addItem("reference", label, dim.getDisplayValue(value), value, magenta, false, getDateFromBuildID(buildID, true), false); - continue; - } - } - if (lastSevenNightlyBuildNames(t.getTimeSeriesLabels(), current).contains(buildID)) { - graph.addItem("main", buildID, dim.getDisplayValue(value), value, yellow, false, getDateFromBuildID(buildID), false); - continue; - } else if (buildID.startsWith("N")) - continue; - - for (int i=0;i. - */ - public static String getImageMap(LineGraph p, String imageSource, String rawDataFileLink) { - String result = ""; - String areas = p.getAreas(); - - result = result.concat(""); - result = result.concat(""); - result = result.concat(areas); - result = result.concat("\n"); - - return result; - } - - /** - * A utility which returns the index of a given buildId in an array. - * - * @param timeSeriesLabels - - * array of buildIds - * @param buildId - - * the buildId for which to find return value. - * @return The index of the buildID in the array. - */ - public static int getBuildNameIndex(String[] timeSeriesLabels, String buildId) { - for (int i = timeSeriesLabels.length - 1; i > -1; i--) { - String timeSeriesLabel = timeSeriesLabels[i]; - if (timeSeriesLabel.startsWith(buildId)) - return i; - } - return -1; - } - - /** * Downsample Image to 8 bit depth format so that the resulting image data * can be saved to GIF. Note. If the source image contains photo quality * content with more than 256 colours, resulting data will look very poor. @@ -646,141 +283,14 @@ return -1; } - public static void printVariabilityTable(Hashtable rawDataTables, String outputFile, String[] configList) { - Hashtable configs=new Hashtable(); - for (int i=0;i\n"); - out.println("

Summary of Elapsed Process Variation Coefficients

\n"+ - "

This table provides a bird's eye view of variability in elapsed process times\n"+ - "for baseline and current build stream performance scenarios." + - " This summary is provided to facilitate the identification of scenarios that should be examined due to high variability." + - "The variability for each scenario is expressed as a coefficient\n"+ - "of variation (CV). The CV is calculated by dividing the standard deviation\n"+ - "of the elapse process time over builds by the average elapsed process\n"+ - "time over builds and multiplying by 100.\n"+ - "

High CV values may be indicative of any of the following:

\n"+ - "
  1. an unstable performance test.
    2. \n"+ - "
    • may be evidenced by an erratic elapsed process line graph.

    \n"+ - "
  2. performance regressions or improvements at some time in the course of builds.
    3. \n"+ - "
    • may be evidenced by plateaus in elapsed process line graphs.

    \n"+ - "
  3. unstable testing hardware.\n" + - "
    • consistent higher CV values for one test configuration as compared to others across" + - " scenarios may be related to hardward problems.
\n"+ - "

Scenarios are listed in alphabetical order in the far right column. A scenario's\n"+ - "variation coefficients (CVs) are in columns to the left for baseline and current\n"+ - "build streams for each test configuration. Scenarios with CVs > 10% are highlighted\n"+ - "in yellow (10%<CV<20%) and orange(CV>20%).

\n"+ - "

Each CV value links to the scenario's detailed results to allow viewers to\n"+ - "investigate the variability.

\n"); - - Hashtable cvTable = (Hashtable) rawDataTables.get(scenarios[0]); - String[] configNames = (String[]) cvTable.keySet().toArray(new String[cvTable.size()]); - Arrays.sort(configNames); - - - int configColumns=configNames.length/2; - out.println("
Baseline CVsCurrent Build Stream CVsScenario Name
n/a"); + } else if (variation >= 20) { + stream.print(""); + } else { + stream.print(""); + } + stream.print(""); + stream.print(variation); + stream.print("%
" + - ""+ - ""+ - ""+ - ""); - - - for (int i = 0; i < configNames.length; i++) { - //configNames here have prefix cConfig- or bConfig- depending on whether the data comes from - //current build stream data or baseline data. - out.print(""); - } - out.println("\n"); - - for (int i = 0; i < scenarios.length; i++) { - Hashtable aCvTable = (Hashtable) rawDataTables.get(scenarios[i]); - String scenario = scenarios[i]; - String scenarioFile=scenario.replace('#', '.').replace(':', '_').replace('\\', '_')+".html"; - - for (int j = 0; j < configNames.length; j++) { - String url=configNames[j].substring(8)+"/"+scenarioFile; - if (aCvTable.get(configNames[j]) == null) { - out.print(""); - continue; - } - String displayValue = aCvTable.get(configNames[j]).toString(); - if (displayValue==null){ - out.print(""); - continue; - } - try { - double value = Double.parseDouble(displayValue.substring(0, displayValue.length() - 1)); - if (value > 10 && value < 20) - out.print(""); - else if (value >= 20) - out.print(""); - else - out.print(""); - } catch (NumberFormatException e) { - e.printStackTrace(); - out.print(""); - } - } - out.println(""); - out.println("\n"); - } - } catch (Exception e) { - e.printStackTrace(); - }finally{ - out.println("
Baseline CVsCurrent Build Stream CVsScenario Name
" + ((ConfigDescriptor)configDescriptors.get(configNames[i].substring(8))).description + "
n/an/a" + displayValue + "" + displayValue + "" + displayValue + "n/a" + scenario + "
"); - out.flush(); - out.close(); - } - } - - public static double[] resultsStatistics(TimeSeries timeSeries) { - try { - double valueRef = timeSeries.getValue(0), value = timeSeries.getValue(1); - long countRef = timeSeries.getCount(0), count = timeSeries.getCount(1); - double stddevRef = timeSeries.getStddev(0), stddev = timeSeries.getStddev(1); - double stderr = (countRef == 1 || count == 1) - ? Double.NaN - : (Double.isNaN(stddevRef) - ? Math.sqrt((stddev * stddev / count)) / valueRef - : Math.sqrt((stddevRef * stddevRef / countRef) + (stddev * stddev / count)) / valueRef); - return new double[] { - (value - valueRef) / valueRef, - stderr, - }; - } - catch (ArrayIndexOutOfBoundsException aioobe) { - return null; - } - } - - public static boolean hasConfidentResult(TimeSeries timeSeries) { - double[] resultStats = resultsStatistics(timeSeries); - return (confidenceLevel(resultStats) & ERR) == 0; - } - public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) { - String current = (String) variations.get(PerformanceTestPlugin.BUILD); - Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS; - Scenario newScenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, baseline, current, new Dim[] { significanceDimension }); - TimeSeries timeSeries = newScenario.getTimeSeries(significanceDimension); - double[] results = resultsStatistics(timeSeries); - return failureMessage(results, true); - } + + /** + * Returns a message corresponding to given statistics. + * + * @param resultStats The value with its standard error + * @param full + * @return The failure message. May be empty if stats are good... + */ public static String failureMessage(double[] resultStats, boolean full) { StringBuffer buffer = new StringBuffer(); int level = confidenceLevel(resultStats); @@ -816,6 +326,19 @@ } return buffer.toString(); } + + /** + * Returns the confidence level for given statistics: + * + * + * @param resultStats array of 2 doubles, the former is the average value and + * the latter is the standard error made while computing the average. + * @return a value telling caller the level of confidence of the provided value + */ public static int confidenceLevel(double[] resultStats) { int level = OK; if (resultStats != null){ @@ -832,38 +355,15 @@ } return level; } - public static boolean matchPattern(String name, String pattern) { - if (pattern.equals("%")) return true; - if (pattern.indexOf('%') >= 0 || pattern.indexOf('_') >= 0) { - StringTokenizer tokenizer = new StringTokenizer(pattern, "%_", true); - int start = 0; - String previous = ""; - while (tokenizer.hasMoreTokens()) { - String token = tokenizer.nextToken(); - if (!token.equals("%") && !token.equals("_")) { - if (previous.equals("%")) { - int idx = name.substring(start).indexOf(token); - if (idx < 0) return false; - start += idx; - } else if (previous.equals("_")) { - if (!name.substring(++start).startsWith(token)) return false; - } - start += token.length(); - } - previous = token; - } - if (previous.equals("%")) { - return true; - } else if (previous.equals("_")) { - return name.length() == start; - } - return name.endsWith(previous); - } - return name.equals(pattern); - } - public static String getImage(int confidence, boolean hasExplanation) { - boolean scenarioFailed = (confidence & DEV) != 0; + /** + * Get an icon image corresponding to a given level of confidence and explanation. + * + * @param confidence the confiden level + * @param hasExplanation flags indicates whether the confidence may be tempered by an explanation + * @return Corresponding image + */ + public static String getImage(int confidence, boolean scenarioFailed, boolean hasExplanation) { String image = null; if (scenarioFailed) { @@ -884,72 +384,4 @@ return image; } - public static boolean hasSummary(SummaryEntry[] summaries, String scenarioName) { - int length = summaries == null ? 0 : summaries.length; - for (int i=0; i= 0; - if (!hasClassName) { - testSeparator = scenarioName.lastIndexOf('.'); - if (testSeparator <= 0) { - if (max > 0 && scenarioName.length() > max) { - return "*"+scenarioName.substring(0, max); - } - return scenarioName; - } - } - int classSeparator = scenarioName.substring(0, testSeparator).lastIndexOf('.'); - if (classSeparator < 0) { - if (max > 0 && scenarioName.length() > max) { - return "*"+scenarioName.substring(0, max); - } - return scenarioName; - } - int length = scenarioName.length(); - String shortName = scenarioName.substring(classSeparator+1, length); - if (!hasClassName && shortName.startsWith("test.")) { // specific case for swt... - shortName = shortName.substring(5); - } - - // Remove qualification from test name - StringTokenizer tokenizer = new StringTokenizer(shortName, " :,", true); - StringBuffer buffer = new StringBuffer(tokenizer.nextToken()); - while (tokenizer.hasMoreTokens()) { - String token = tokenizer.nextToken(); - char fc = token.charAt(0); - while (fc == ' ' || fc == ',' || fc == ':') { - buffer.append(token); // add the separator - token = tokenizer.nextToken(); - fc = token.charAt(0); - } - int last = token .lastIndexOf('.'); - if (last >= 3) { - int first = token .indexOf('.'); - if (first == last) { - buffer.append(token); - } else { -// buffer.append(token.substring(0, first)); -// buffer.append("..."); - buffer.append(token.substring(last+1)); - } - } else { - buffer.append(token); - } - } - if (max > 0 && buffer.length() > max) { - return "*"+buffer.substring(0, max); - } - return buffer.toString(); - } } \ No newline at end of file Index: src/org/eclipse/test/performance/ui/ScenarioResults.java =================================================================== RCS file: src/org/eclipse/test/performance/ui/ScenarioResults.java diff -N src/org/eclipse/test/performance/ui/ScenarioResults.java --- src/org/eclipse/test/performance/ui/ScenarioResults.java 5 Jun 2007 13:24:06 -0000 1.35 +++ /dev/null 1 Jan 1970 00:00:00 -0000 @@ -1,376 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2004, 2007 IBM Corporation and others. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * IBM Corporation - initial API and implementation - *******************************************************************************/ -package org.eclipse.test.performance.ui; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Hashtable; -import java.util.Iterator; - -import junit.framework.AssertionFailedError; - -import org.eclipse.test.internal.performance.data.Dim; -import org.eclipse.test.internal.performance.db.Scenario; -import org.eclipse.test.internal.performance.db.TimeSeries; -import org.eclipse.test.internal.performance.db.Variations; -import org.eclipse.test.performance.ui.Utils.ConfigDescriptor; - -public class ScenarioResults { - private Scenario[] scenarios; - private String baseline; - private String baselinePrefix = null; - private String current; - private ArrayList pointsOfInterest; - private ArrayList buildIDStreamPatterns; - private Hashtable scenarioComments; - private Hashtable variabilityData; - private ConfigDescriptor configDescriptor; - private String outputDir; - private HashMap scenarioMap; - private Variations variations; - /** - * Summary of results for a scenario for a given build compared to a - * reference. - * - * @param scenarios - - * the array of Scenario objects for which to generate results. - * @param baseline - - * the reference build ID - * @param current - - * the current buildID - * @param configDescriptor - - * a ConfigDescriptor object. - * @param pointsOfInterest - - * an ArrayList of buildId's to highlight on line graphs. - */ - public ScenarioResults(Utils.ConfigDescriptor configDescriptor, Scenario[] scenarios, String baseline, String baselinePrefix, String current, - ArrayList pointsOfInterest, Hashtable scenarioComments, ArrayList buildIDPatterns, Hashtable variabilityTable, String outputDir,Variations variations) { - - this.scenarios = scenarios; - this.variations=variations; - scenarioMap=new HashMap(); - for (int i=0;i" + t.getScenarioName() + "(" + configDescriptor.description + ")" + ""); //$NON-NLS-1$ - ps.println("

Scenario: " + t.getScenarioName() + " (" + configDescriptor.description + ")


"); //$NON-NLS-1$ //$NON-NLS-2$ - - String failureMessage = Utils.failureMessage(variations, t.getScenarioName(),baseline,configDescriptor.name); - if (failureMessage != null){ - ps.println("
"+failureMessage+"
\n"); - } - - String comment= (String)scenarioComments.get(t.getScenarioName()); - if (comment != null) { - ps.println("

Note:
\n"); - ps.println(comment + "

"); - } - - //print link to raw data. - ps.println("

Raw data and Stats

\n"); - ps.println("Click measurement name to view line graph of measured values over builds.

\n"); - ps.println(""); //$NON-NLS-1$ //$NON-NLS-2$ - - Dim[] dimensions = filteredDimensions(t.getDimensions()); - try { - ps.println(""); //$NON-NLS-1$ - for (int i = 0; i < dimensions.length; i++) { - Dim dim = dimensions[i]; - String dimName = dim.getName(); - ps.println(""); - } - ps.print("\n"); - - // store current and reference values for diff later - double[][] diffValues = new double[2][dimensions.length]; - // to determine if diff is possible - boolean[] refValueExistance = new boolean[dimensions.length]; - - for (int j = 0; j < 2; j++) { - String referenceIndicator = (j == 1) ? "(reference)" : ""; - - String buildName = ((buildNameIndeces[j] == -1) ? "n/a" : t.getTimeSeriesLabels()[buildNameIndeces[j]]) + referenceIndicator; - ps.print(""); - - for (int i = 0; i < dimensions.length; i++) { - Dim dim = dimensions[i]; - - TimeSeries ts = t.getTimeSeries(dim); - if (j == 1 && buildNameIndeces[j] != -1) - refValueExistance[i] = true; - - if (buildNameIndeces[j] != -1) - diffValues[j][i] = ts.getValue(buildNameIndeces[j]); - - String displayValue = (buildNameIndeces[j] == -1) ? "n/a" : dim.getDisplayValue(ts.getValue(buildNameIndeces[j])); - String stddev = (buildNameIndeces[j] == -1) ? "0" : dim.getDisplayValue(ts.getStddev(buildNameIndeces[j])); - - if (stddev.startsWith("0 ") || stddev.equals("0")) - ps.println(""); - else - ps.println(""); - } - ps.print(""); - } - // get diffs and print results - ps.println(getDiffs(diffValues, dimensions, refValueExistance)); - ps.println(); - ps.println("
Build Id" + dimName + "
" + buildName + "" + displayValue + "" + displayValue + " [" + stddev + "]" + "
"); - ps.println("*Delta values in red and green indicate degradation > 10% and improvement > 10%,respectively.

"); - ps.println("
\n\n"); - - // print text legend. - ps.println("Black and yellow points plot values measured in integration and last seven nightly builds.
\n" + "Magenta points plot the repeated baseline measurement over time.
\n" - + "Boxed points represent previous releases, milestone builds, current reference and current build.

\n" - + "Hover over any point for build id and value.\n"); - - // print image maps of historical - for (int i = 0; i < dimensions.length; i++) { - Dim dim = dimensions[i]; - String dimName = dim.getName(); - - TimeLineGraph lg = Utils.getLineGraph(t, dim.getName(), baseline, baselinePrefix, current, pointsOfInterest,buildIDStreamPatterns); - - String lgImg = outputDir + "/graphs/" + scenarioFileName + "_" + dimName + ".gif"; - Utils.printLineGraphGif(lg, lgImg); - ps.println("
"); - ps.println("
" + dimName + "
"); - ps.println(Utils.getDimensionDescription(dimName) + "

\n"); - ps.println(Utils.getImageMap(lg, "graphs/" + scenarioFileName + "_" + dimName + ".gif",scenarioFileName+"_raw.html")); - } - ps.println("

"); - ps.println(Utils.HTML_CLOSE); //$NON-NLS-1$ - if (ps != System.out) - ps.close(); - - } catch (AssertionFailedError e) { - e.printStackTrace(); - continue; - } - } - } - - private void printDetails() { - - String outFile = null; - PrintStream ps = null; - - for (int s = 0; s < scenarios.length; s++) { - Scenario t = scenarios[s]; - // don't produce result if none exists for current build - if (Utils.getBuildNameIndex(t.getTimeSeriesLabels(), current) == -1) { - continue; - } - Dim[] dimensions = filteredDimensions(t.getDimensions()); - //store cv for aggregate data - Hashtable data=(Hashtable)variabilityData.get(t.getScenarioName()); - if (data==null){ - data=new Hashtable(); - variabilityData.put(t.getScenarioName(),data); - } - - RawDataTable currentResultsTable=new RawDataTable(t,dimensions,buildIDStreamPatterns,current); - data.put("cConfig-"+configDescriptor.name,currentResultsTable.getCV()); - - //create table for baseline data - RawDataTable baselineResultsTable; - if (baselinePrefix!=null) - baselineResultsTable=new RawDataTable(t,dimensions,baselinePrefix,baseline); - else - baselineResultsTable=new RawDataTable(t,dimensions,baseline,baseline); - - data.put("bConfig-"+configDescriptor.name,baselineResultsTable.getCV()); - - - String scenarioFileName = t.getScenarioName().replace('#', '.').replace(':', '_').replace('\\', '_'); - outFile = outputDir + "/" + scenarioFileName + "_raw.html"; - if (outFile != null) { - try { - new File(outFile).getParentFile().mkdirs(); - ps = new PrintStream(new BufferedOutputStream(new FileOutputStream(outFile))); - } catch (FileNotFoundException e) { - System.err.println("can't create output file" + outFile); //$NON-NLS-1$ - } - } - if (ps == null) - ps = System.out; - ps.println(Utils.HTML_OPEN); - ps.println(Utils.HTML_DEFAULT_CSS); - ps.println("" + t.getScenarioName() + "(" + configDescriptor.description + ")" + " - Details"); //$NON-NLS-1$ - ps.println("

Scenario: " + t.getScenarioName() + " (" + configDescriptor.description + ")

"); //$NON-NLS-1$ - ps.println("VIEW GRAPH

"); //$NON-NLS-1$ - ps.println("\n"); - ps.println("" + - "" - +""); - ps.println("
Current Stream Test RunsBaseline Test Runs
"+currentResultsTable.toHtmlString()+""+baselineResultsTable.toHtmlString()+"
"); - ps.close(); - } - } - - private String getMostRecentMatchingBuildID(Scenario scenario, String buildIdPrefix) { - String[] buildIds = scenario.getTimeSeriesLabels(); - for (int i = buildIds.length - 1; i > -1; i--) { - if (buildIds[i].startsWith(buildIdPrefix)) - return buildIds[i]; - } - return null; - } - - private ArrayList getAllMatchingBuildIds(Scenario scenario, String buildIdPrefix) { - ArrayList result = new ArrayList(); - String[] buildIds = scenario.getTimeSeriesLabels(); - for (int i = buildIds.length - 1; i > -1; i--) { - if (buildIds[i].startsWith(buildIdPrefix) && !buildIds[i].equals(baseline)) - result.add(buildIds[i]); - } - return result; - } - - private String getDiffs(double[][] values, Dim[] dimensions, boolean[] refValueExistance) { - String diffRow = "*Delta"; - for (int j = 0; j < dimensions.length; j++) { - Dim dim = dimensions[j]; - - double diffValue = values[0][j] - values[1][j]; - double diffPercentage = 0; - if (values[1][j] != 0) - diffPercentage = ((int) (((diffValue / Math.abs(values[1][j])) * 1000))) / 10.0; - String diffDisplayValue = dim.getDisplayValue(diffValue); - // green - String fontColor = ""; - if ((diffPercentage < -10 && !dim.largerIsBetter()) || (diffPercentage > 10 && dim.largerIsBetter())) - fontColor = "#006600"; - if ((diffPercentage < -10 && dim.largerIsBetter()) || (diffPercentage > 10 && !dim.largerIsBetter())) - fontColor = "#FF0000"; - - diffPercentage = Math.abs(diffPercentage); - String percentage = (diffPercentage == 0) ? "" : "
" + diffPercentage + " %"; - - if (diffPercentage > 10 || diffPercentage < -10) { - diffRow = diffRow.concat("" + diffDisplayValue + percentage + ""); - } else if (refValueExistance[j]) { - diffRow = diffRow.concat("" + diffDisplayValue + percentage + ""); - } else { - diffRow = diffRow.concat("n/a"); - } - } - diffRow = diffRow.concat(""); - return diffRow; - } - - private Dim[] filteredDimensions(Dim[] dimensions) { - ArrayList list = new ArrayList(); - ArrayList filtered = new ArrayList(); - list.add(0, "Elapsed Process"); - list.add(1, "CPU Time"); - list.add(2, "Invocation Count"); - list.add(3, "GDI Objects"); - //list.add(4, "Kernel time"); - //list.add(5, "Data Size"); - //list.add(6, "Library Size"); - //list.add(7, "Text Size"); - // list.add(8,"Used Java Heap"); - // list.add(9,"Committed"); - // list.add(10,"Page Faults"); - // list.add(11,"Hard Page Faults"); - // list.add(12,"Soft Page Faults"); - - for (int i = 0; i < dimensions.length; i++) { - String dimName = dimensions[i].getName(); - if (list.contains(dimName)) - list.set(list.indexOf(dimName), dimensions[i]); - } - Iterator iterator = list.iterator(); - while (iterator.hasNext()) { - Object tmp = iterator.next(); - try { - if ((Dim) tmp instanceof Dim) - filtered.add(tmp); - } catch (ClassCastException e) { - //silently ignore - } - } - return (Dim[]) filtered.toArray(new Dim[filtered.size()]); - } - -} Index: src/org/eclipse/test/performance/ui/ScenarioStatusTable.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/ScenarioStatusTable.java,v retrieving revision 1.32 diff -u -r1.32 ScenarioStatusTable.java --- src/org/eclipse/test/performance/ui/ScenarioStatusTable.java 5 Jun 2007 17:41:07 -0000 1.32 +++ src/org/eclipse/test/performance/ui/ScenarioStatusTable.java 31 Aug 2007 15:14:57 -0000 @@ -11,292 +11,206 @@ package org.eclipse.test.performance.ui; import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Hashtable; +import java.util.List; import java.util.StringTokenizer; -import org.eclipse.test.internal.performance.PerformanceTestPlugin; -import org.eclipse.test.internal.performance.data.Dim; -import org.eclipse.test.internal.performance.db.DB; -import org.eclipse.test.internal.performance.db.Scenario; -import org.eclipse.test.internal.performance.db.SummaryEntry; -import org.eclipse.test.internal.performance.db.TimeSeries; -import org.eclipse.test.internal.performance.db.Variations; -import org.eclipse.test.performance.Dimension; - +import org.eclipse.test.internal.performance.results.BuildResults; +import org.eclipse.test.internal.performance.results.ConfigResults; +import org.eclipse.test.internal.performance.results.PerformanceResults; +import org.eclipse.test.internal.performance.results.ScenarioResults; + +/** + * Class used to print a scenario status table. + */ public class ScenarioStatusTable { - - private Hashtable configMaps=null; - private Variations variations; - private String scenarioPattern; - private ArrayList configNames=new ArrayList(); - private Hashtable scenarioComments; - private SummaryEntry[] fingerprintEntries; - private final String baseline; - - private class ScenarioStatus{ - Hashtable statusMap; - String name, shortName; - Hashtable configStatus; - Hashtable resultsMap; - boolean hasSlowDownExplanation=false; - boolean fingerprint = false; - boolean hasBaseline = true; - - public ScenarioStatus(String scenarioName){ - name = scenarioName; - statusMap = new Hashtable(); - configStatus = new Hashtable(); - resultsMap = new Hashtable(); - } - - } - - /** - * Creates an HTML table of red x/green check for a scenario for each - * configuration. - * - * @param variations - * @param scenarioPattern - * @param configDescriptors - */ - public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, SummaryEntry[] fpSummaries, String baseline){ - configMaps=configDescriptors; - this.variations=variations; - this.scenarioPattern=scenarioPattern; - this.scenarioComments=scenarioComments; - this.baseline= baseline; - this.fingerprintEntries = fpSummaries; - } - - /** - * Prints the HTML representation of scenario status table into the given stream. - */ - public void print(PrintStream stream, boolean filter) { - String OS="config"; - Scenario[] scenarios= DB.queryScenarios(variations, scenarioPattern, OS, null); - - if (scenarios != null && scenarios.length > 0) { - ArrayList scenarioStatusList=new ArrayList(); - - for (int i= 0; i < scenarios.length; i++) { - Scenario scenario= scenarios[i]; - String scenarioName=scenario.getScenarioName(); - if (filter && !Utils.matchPattern(scenarioName, scenarioPattern)) continue; - // returns the config names. Making assumption that indices in - // the configs array map to the indices of the failure messages. - String[] configs=scenario.getTimeSeriesLabels(); - String[] failureMessages= scenario.getFailureMessages(); - ScenarioStatus scenarioStatus=new ScenarioStatus(scenarioName); - scenarioStatus.fingerprint = Utils.hasSummary(this.fingerprintEntries, scenarioName); - String scenarioComment= (String)scenarioComments.get(scenarioName); - if (scenarioComment != null) - scenarioStatus.hasSlowDownExplanation= true; - - int confsLength = configs.length; - for (int j=0; j(vs. "); - shortName.append(timeSeriesLabels[0]); - shortName.append(")"); - scenarioStatus.shortName = shortName.toString(); - } - } + private String component; + private PrintStream stream; + private int jsIdCount; + +/** + * Creates an HTML table of red x/green check for a scenario for each + * configuration. + */ +public ScenarioStatusTable(String name, PrintStream stream) { + this.component = name; + this.stream = stream; +} - // Store scenario status - if (!scenarioStatusList.contains(scenarioStatus)) { - scenarioStatusList.add(scenarioStatus); - } - } - } - - String label=null; - stream.println("

Scenario Status

"); - stream.println("The following table gives a complete but compact view of performance results for the component.
"); - stream.println("Each line of the table shows the results for one scenario on all machines.

"); - stream.println("The name of the scenario is in bold when its results are also displayed in the fingerprints
"); - stream.println("and starts with an '*' when the scenario has no results in the last baseline run.

"); - stream.println("Here are information displayed for each test (ie. in each cell):"); - stream.println("
    "); - stream.println("
  • an icon showing whether the test fails or passes and whether it's reliable or not.
    "); - stream.println("The legend for this icon is:"); - stream.println("
      "); - stream.print("
    • Green (): mark a successful result, which means this test has neither significant performance regression nor significant standard error
      • "); - stream.print("
    • Red (): mark a failing result, which means this test shows a significant performance regression (more than 10%)
      • "); - stream.print("
    • Gray (): mark a failing result (see above) with a comment explaining this degradation.
      • "); - stream.print("
    • Yellow ( or ): mark a failing or successful result with a significant standard error (more than "); - stream.print(Utils.STANDARD_ERROR_THRESHOLD_STRING); - stream.println(")
      • "); - stream.print("
    • Black (): mark an undefined result, which means that deviation on this test is not a number (NaN) or is infinite (happens when the reference value is equals to 0!)
      • "); - stream.println("
    • \"n/a\": mark a test for with no performance results
      • "); - stream.println("
    • "); - stream.println("
  • the value of the deviation from the baseline as a percentage (ie. formula is: (build_test_time - baseline_test_time) / baseline_test_time)
    • "); - stream.println("
  • the value of the standard error of this deviation as a percentage (ie. formula is: sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time)
    "); - stream.println("When test only has one measure, the standard error cannot be computed and is replaced with a '[n/a]'.
    • "); - stream.println("
"); - stream.println("Hints:
    "); - stream.println("
  • fly over image of failing tests to see the complete error message
    • "); - stream.println("
  • to look at the complete and detailed test results, click on its image
    • "); - stream.println("
"); - stream.println(); - stream.println(""); - stream.println(""); - stream.print(""); +/** + * Prints the HTML representation of scenario status table into the given stream. + */ +public void print(PerformanceResults performanceResults) { + printTitle(); + List scenarii = performanceResults.getComponentScenarii(this.component); + String baselineName = performanceResults.getBaselineName(); + int size = scenarii.size(); + printColumnsTitle(size, performanceResults); + this.jsIdCount = 0; + for (int i=0; i"); + this.stream.print(""); + return; + } + BuildResults currentBuildResults = configResults.getCurrentBuildResults(); + String failure = currentBuildResults.getFailure(); + double[] deviation = configResults.getCurrentBuildDeviation(); + int confidence = Utils.confidenceLevel(deviation); + boolean hasFailure = failure != null; + String comment = currentBuildResults.getComment(); + String image = Utils.getImage(confidence, hasFailure, comment != null); + this.stream.print(""); } + stream.print(buffer.toString()); +} - private String htmlDimensionColumnHeaders() { - String result=""; - for (int i=0;i"+dimensions[i].getName()+""); +/* + * Print all build results in the table. + */ +private void printDetails() { + stream.print(""); + printColumnHeaders(); + stream.println(""); + + List builds = this.configResults.getBuildsMatchingPrefixes(this.buildPrefixes); + Collections.reverse(builds); + int size = builds.size(); + for (int i=0; i"); + int dimLength = this.dimensions.length; + for (int d=0; d"); } - return result; + if (debug) System.out.println(); + stream.println(""); } + if (debug) System.out.println("\n"); +} - private String htmlDetails() { - String result=""; - result = result.concat(htmlDimensionColumnHeaders()); - result = result.concat("\n"); - - Set sampleKeys = buildIDTable.keySet(); - String[] buildIDs = sortBuildIDsByDate((String[]) sampleKeys.toArray(new String[buildIDTable.size()])); - - for (int i = buildIDs.length; i > 0; i--) { - String buildID = buildIDs[i-1]; - if (buildID == null) - continue; - - Hashtable values=(Hashtable)buildIDTable.get(buildID); - if (values==null) - continue; - - result = result.concat("\n"); - - for (int j=0;j" + dimensions[j].getDisplayValue(value) + ""); - } - } - result=result.concat("\n"); - return result; +/* + * Print summary on top of the table. + */ +private void printSummary() { + stream.print(""); + printColumnHeaders(); + stream.println(""); + + int length = this.dimensions.length; + double[][] dimStats = new double[2][]; + for (int i=0; i"); + for (int i=0; i"); + } + stream.println(""); + stream.print(""); + printRowDoubles(dimStats, 1); + stream.println(""); + stream.print(""); + printRowDoubles(dimStats, 2); + stream.println(""); + stream.print(""); + printRowDoubles(dimStats, 3); + stream.println(""); + + // Blank line + stream.print(""); + for (int i=0; i "); } + stream.println(""); +} - private String htmlSummary() { - if (derivedValues==null) - return ""; - //print summary values - //print totals - String result=""; - result = result.concat(htmlDimensionColumnHeaders()); - result = result.concat("\n"); - - //print sample counts - - result = result.concat("\n"); - result = result.concat(htmSummaryRow(SAMPLECOUNT)); - - //print averages - result = result.concat("\n"); - result = result.concat(htmSummaryRow(MEAN)); - - //print standard deviation - result = result.concat("\n"); - result = result.concat(htmSummaryRow(STDDEV)); - - //print coefficient of variation - result = result.concat("\n"); - result = result.concat(htmSummaryRow(CV)); - - result = result.concat("\n"); - for (int i=0;i "); - } - result=result.concat("\n"); - - return result; - } - - private String htmSummaryRow(String summaryType) { - String result=""; - Hashtable tmp; - for (int j=0;j10&&value<20) - result = result.concat(""); - else if (value>=20) - result = result.concat(""); - else - result = result.concat(""); - } else - result= result.concat(""); +/* + * Print values in table row. + */ +private void printRowDoubles(double[][] stats, int idx) { + int length = this.dimensions.length; + for (int i=0; i10 && value<20) { + stream.print(""); + } else { + printDimTitle(dimName); + stream.print(this.dimensions[i].getDisplayValue(value)); + stream.print(""); } - return result; } +} - public String getCV() { - Hashtable tmp; - String dimensionName="Elapsed Process"; - tmp=(Hashtable)(derivedValues.get(dimensionName)); - if (tmp==null) - return "n/a"; - if (tmp.get(CV)!=null) - return tmp.get(CV).toString(); - return "n/a"; - - } +/* + * Print dim title inside value reference. + * TODO (frederic) See if this title is really necessary + */ +private void printDimTitle(String dimName) { + stream.print(""); + Dim[] dimensions = AbstractResults.SUPPORTED_DIMS; + int dimLength = dimensions.length; + for (int d=0; d"); + stream.print(displayValue); + if (stddev < 0) { + stream.println(" [n/a]"); + } else if (stddev > 0) { + stream.print(" ["); + stream.print(dimensions[d].getDisplayValue(stddev)); + stream.print("]"); + } + stream.print( ""); + } + stream.println(""); +} + +/* + * Print the line showing the difference between current and baseline builds. + */ +private void printDifferenceLine(PrintStream stream, ConfigResults configResults) { + stream.print(""); + Dim[] dimensions = AbstractResults.SUPPORTED_DIMS; + int dimLength = dimensions.length; + for (int d=0; d 10 && currentDim.largerIsBetter())) + fontColor = "#006600"; + if ((diffPercentage < -10 && currentDim.largerIsBetter()) || (diffPercentage > 10 && !currentDim.largerIsBetter())) + fontColor = "#FF0000"; + + diffPercentage = Math.abs(diffPercentage); + String percentage = (diffPercentage == 0) ? "" : "
" + diffPercentage + " %"; + + if (diffPercentage > 10 || diffPercentage < -10) { + stream.print(""); + } else { + stream.print(""); + } + } + stream.print(""); +} + +/* + * Returns a LineGraph object representing measurements for a scenario over builds. + */ +private TimeLineGraph getLineGraph(ScenarioResults scenarioResults, ConfigResults configResults, Dim dim, List highlightedPoints, List currentBuildIdPrefixes) { + Display display = Display.getDefault(); + + Color black = display.getSystemColor(SWT.COLOR_BLACK); + Color yellow = display.getSystemColor(SWT.COLOR_DARK_YELLOW); + Color magenta = display.getSystemColor(SWT.COLOR_MAGENTA); + + String scenarioName = scenarioResults.getName(); + TimeLineGraph graph = new TimeLineGraph(scenarioName + ": " + dim.getName(), dim); + String baseline = configResults.getBaselineBuildName(); + String current = configResults.getCurrentBuildName(); + + Iterator builds = configResults.getResults(); + List lastSevenNightlyBuilds = configResults.lastNightlyBuildNames(7); + buildLoop: while (builds.hasNext()) { + BuildResults buildResults = (BuildResults) builds.next(); + String buildID = buildResults.getName(); + int underscoreIndex = buildID.indexOf('_'); + String label = (underscoreIndex != -1 && (buildID.equals(baseline) || buildID.equals(current))) ? buildID.substring(0, underscoreIndex) : buildID; + + double value = buildResults.getValue(dim.getId()); + + if (buildID.equals(current)) { + Color color = black; + if (buildID.startsWith("N")) + color = yellow; + + graph.addItem("main", label, dim.getDisplayValue(value), value, color, true, Utils.getDateFromBuildID(buildID), true); + continue; + } + if (highlightedPoints.contains(buildID)) { + graph.addItem("main", label, dim.getDisplayValue(value), value, black, false, Utils.getDateFromBuildID(buildID, false), true); + continue; + } + if (buildID.charAt(0) == 'N') { + if (lastSevenNightlyBuilds.contains(buildID)) { + graph.addItem("main", buildID, dim.getDisplayValue(value), value, yellow, false, Utils.getDateFromBuildID(buildID), false); + } + continue; + } + for (int i=0;i" + scenarioName + "(" + configBox + ")" + " - Details"); //$NON-NLS-1$ + stream.println("

Scenario: " + scenarioName + " (" + configBox + ")

"); //$NON-NLS-1$ + stream.println("VIEW GRAPH

"); //$NON-NLS-1$ + stream.println("

All "); - stream.print(scenarios.length); - stream.println(" scenarios

"); + boolean hasSummary = scenarioResults.hasSummary(); + if (hasSummary) this.stream.print(""); + String scenarioBaseline = scenarioResults.getBaselineBuildName(); + boolean hasBaseline = baselineName.equals(scenarioBaseline); + if (!hasBaseline) { + this.stream.print("*"); + this.stream.print(scenarioResults.getShortName()); + this.stream.print(" (vs. "); + this.stream.print(scenarioBaseline); + this.stream.print(")"); + } else { + this.stream.print(scenarioResults.getShortName()); + } + if (hasSummary) this.stream.print(""); + this.stream.println(); + String[] configs = performanceResults.getConfigNames(true/*sort*/); + int length = configs.length; + for (int j=0; j"); +} - for (int i= 0; i < configNames.size(); i++){ - label = configNames.get(i).toString(); - String columnTitle = label; - if (configMaps!=null) { - Utils.ConfigDescriptor configDescriptor= (Utils.ConfigDescriptor)configMaps.get(label); - if (configDescriptor != null) { - int idx = configDescriptor.description.indexOf('('); - if (idx < 0) { - columnTitle = configDescriptor.description; - } else { - // first line - StringTokenizer tokenizer = new StringTokenizer(configDescriptor.description.substring(0, idx).trim(), " "); - StringBuffer buffer = new StringBuffer(tokenizer.nextToken()); - while (tokenizer.hasMoreTokens()) { - buffer.append(" "); - buffer.append(tokenizer.nextToken()); - } - buffer.append(' '); - // second line - tokenizer = new StringTokenizer(configDescriptor.description.substring(idx).trim(), " "); - buffer.append(tokenizer.nextToken()); - while (tokenizer.hasMoreTokens()) { - buffer.append(" "); - buffer.append(tokenizer.nextToken()); - } - columnTitle = buffer.toString(); - } - } - } - stream.print("
"); - stream.print(columnTitle); - stream.println("
"); +/* + * Print the table columns title. + */ +private void printColumnsTitle(int size, PerformanceResults performanceResults) { + this.stream.println(""); + this.stream.println(""); + this.stream.print(""); + String[] configNames = performanceResults.getConfigNames(true/*sort*/); + String[] configBoxes = performanceResults.getConfigBoxes(true/*sort*/); + int length = configNames.length; + for (int i=0; i"); - stream.print("

All "); + this.stream.print(size); + this.stream.println(" scenarios

"); - if (status.fingerprint) stream.print(""); - if (!status.hasBaseline) stream.print("*"); -// stream.print(status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1)); - stream.print(status.shortName); - if (!status.hasBaseline) stream.print(""); - if (status.fingerprint) stream.print(""); - stream.println(); - for (int i=0;i"); - stream.print(""); - } else { - // create message with tooltip text including deviation with error plus failure message - jsIdCount+=1; - stream.print("class=\"tooltipSource\" onMouseover=\"show_element('toolTip"); - stream.print(jsIdCount); - stream.print("')\" onMouseout=\"hide_element('toolTip"); - stream.print(jsIdCount); - stream.print("')\" \nhref=\""); - stream.print(aUrl); - stream.print('/'); - stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_')); - stream.println(".html\">"); - stream.print(""); - stream.print(""); - stream.print(message); - stream.println(""); - } - String result = (String) status.resultsMap.get(configName); - stream.println(result); - }else{ - stream.println(" n/a"); - } - } - stream.flush(); + buffer.append(' '); + // second line + tokenizer = new StringTokenizer(boxName.substring(idx).trim(), " "); + buffer.append(tokenizer.nextToken()); + while (tokenizer.hasMoreTokens()) { + buffer.append(" "); + buffer.append(tokenizer.nextToken()); } - stream.println("
"); + columnTitle = buffer.toString(); } - } + this.stream.print("
"); + this.stream.print(columnTitle); + this.stream.println("
"); + } +} + +/* + * Print the scenario statistics value for the given configuration. + */ +private void printConfigStats(ScenarioResults scenarioResults, String config) { + ConfigResults configResults = scenarioResults.getConfigResults(config); + if (configResults == null || !configResults.isValid()) { + this.stream.print("		n/a"); + this.stream.print(""); + } else { + // create message with tooltip text including deviation with error plus failure message + this.jsIdCount+=1; + this.stream.print("class=\"tooltipSource\" onMouseover=\"show_element('toolTip"); + this.stream.print(jsIdCount); + this.stream.print("')\" onMouseout=\"hide_element('toolTip"); + this.stream.print(jsIdCount); + this.stream.print("')\" \nhref=\""); + this.stream.print(configResults.getName()); + this.stream.print('/'); + this.stream.print(scenarioResults.getFileName()); + this.stream.println(".html\">"); + this.stream.print(""); + this.stream.print(""); + this.stream.print(failure); + this.stream.println(""); + } + String result = Utils.failureMessage(deviation, false); + this.stream.println(result); +} + +/* + * Print the status table explanationtitle. + */ +private void printTitle() { + this.stream.println("

Scenario Status

"); + this.stream.println("The following table gives a complete but compact view of performance results for the component.
"); + this.stream.println("Each line of the table shows the results for one scenario on all machines.

"); + this.stream.println("The name of the scenario is in bold when its results are also displayed in the fingerprints
"); + this.stream.println("and starts with an '*' when the scenario has no results in the last baseline run.

"); + this.stream.println("Here are information displayed for each test (ie. in each cell):"); + this.stream.println("
    "); + this.stream.println("
  • an icon showing whether the test fails or passes and whether it's reliable or not.
    "); + this.stream.println("The legend for this icon is:"); + this.stream.println("
      "); + this.stream.print("
    • Green (): mark a successful result, which means this test has neither significant performance regression nor significant standard error
      • "); + this.stream.print("
    • Red (): mark a failing result, which means this test shows a significant performance regression (more than 10%)
      • "); + this.stream.print("
    • Gray (): mark a failing result (see above) with a comment explaining this degradation.
      • "); + this.stream.print("
    • Yellow ( or ): mark a failing or successful result with a significant standard error (more than "); + this.stream.print(Utils.STANDARD_ERROR_THRESHOLD_STRING); + this.stream.println(")
      • "); + this.stream.print("
    • Black (): mark an undefined result, which means that deviation on this test is not a number (NaN) or is infinite (happens when the reference value is equals to 0!)
      • "); + this.stream.println("
    • \"n/a\": mark a test for with no performance results
      • "); + this.stream.println("
    • "); + this.stream.println("
  • the value of the deviation from the baseline as a percentage (ie. formula is: (build_test_time - baseline_test_time) / baseline_test_time)
    • "); + this.stream.println("
  • the value of the standard error of this deviation as a percentage (ie. formula is: sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time)
    "); + this.stream.println("When test only has one measure, the standard error cannot be computed and is replaced with a '[n/a]'.
    • "); + this.stream.println("
"); + this.stream.println("Hints:
    "); + this.stream.println("
  • fly over image of failing tests to see the complete error message
    • "); + this.stream.println("
  • to look at the complete and detailed test results, click on its image
    • "); + this.stream.println("
"); +} } Index: src/org/eclipse/test/performance/ui/FingerPrint.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/FingerPrint.java,v retrieving revision 1.23 diff -u -r1.23 FingerPrint.java --- src/org/eclipse/test/performance/ui/FingerPrint.java 5 Jun 2007 13:24:06 -0000 1.23 +++ src/org/eclipse/test/performance/ui/FingerPrint.java 31 Aug 2007 15:14:57 -0000 @@ -16,7 +16,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.util.Hashtable; +import java.io.PrintStream; +import java.util.List; import org.eclipse.swt.SWT; import org.eclipse.swt.graphics.GC; @@ -24,236 +25,160 @@ import org.eclipse.swt.graphics.ImageData; import org.eclipse.swt.graphics.ImageLoader; import org.eclipse.swt.widgets.Display; -import org.eclipse.test.internal.performance.PerformanceTestPlugin; -import org.eclipse.test.internal.performance.data.Dim; -import org.eclipse.test.internal.performance.db.DB; -import org.eclipse.test.internal.performance.db.Scenario; -import org.eclipse.test.internal.performance.db.SummaryEntry; -import org.eclipse.test.internal.performance.db.TimeSeries; -import org.eclipse.test.internal.performance.db.Variations; -import org.eclipse.test.performance.ui.Utils.ConfigDescriptor; +import org.eclipse.test.internal.performance.results.AbstractResults; +import org.eclipse.test.internal.performance.results.ConfigResults; +import org.eclipse.test.internal.performance.results.PerformanceResults; +import org.eclipse.test.internal.performance.results.ScenarioResults; + +/** + * Class used to create scenario fingerprint. + */ +public class FingerPrint { +private static final int GRAPH_WIDTH = 1000; -public class FingerPrint { - - private static final int GRAPH_WIDTH= 1000; + String component; + PrintStream stream; + File outputDir; + +public FingerPrint(String name, PrintStream ps, File outputDir) { + if (!name.equals("global")) this.component = name; + this.stream = ps; + this.outputDir = outputDir; +} - String outputDirectory; - String referenceBuildId; - String currentBuildId; - String component; - ConfigDescriptor configDescriptor; - SummaryEntry [] entries; - Variations variations; - BarGraph bar; - String outName; - String title; - Hashtable scenarioComments; - - public FingerPrint() { - } - - - public FingerPrint(String component,ConfigDescriptor config, String reference, String current,Variations variations, String outputDir) { - this.variations=variations; - this.component=component; - referenceBuildId= reference; - currentBuildId= current; - outputDirectory= outputDir; - configDescriptor=config; - variations.put(PerformanceTestPlugin.BUILD, currentBuildId); - if (component==null){ - entries= DB.querySummaries(variations,null); - this.component=""; - } else { - entries=DB.querySummaries(variations,component+'%'); - } - run(); - } - - /** - * Creates the fingerprint gif, image map and scenario status table for the component. - */ - public void run() { - new File(outputDirectory).mkdirs(); - String referenceName=referenceBuildId; - String currentName=currentBuildId; - int referenceUnderscoreIndex=referenceBuildId.indexOf('_'); - int currentUnderscoreIndex=currentBuildId.indexOf('_'); - - if (referenceUnderscoreIndex!=-1) - referenceName=referenceBuildId.substring(0,referenceUnderscoreIndex); - if (currentUnderscoreIndex!=-1) - currentName=currentBuildId.substring(0,currentUnderscoreIndex); - - title="Performance of " + component +" "+currentName + " relative to " + referenceName; - bar= new BarGraph(null); - - if (entries != null) { - for (int i= 0; i < entries.length; i++) { - SummaryEntry summary = entries[i]; - if (summary.comment==null) - add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName); - else{ - setComment(summary.scenarioName, summary.comment); - add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName, summary.comment); - } - } - } - - outName= "FP_" + component+ '_'+referenceName + '_' + currentBuildId+"."+configDescriptor.name; - - if (component=="") - outName= "FP_"+referenceName + '_' + currentBuildId+"."+configDescriptor.name; - save(outputDirectory + '/' + outName); - - //show(bar); - - } - - private void setComment(String scenario, String comment) { - if (scenarioComments==null) - scenarioComments=new Hashtable(); - scenarioComments.put(scenario,comment); - } - - private void add(String name, Dim[] dims, String scenarioName) { - add (name,dims,scenarioName,null); - } - - private void add(String name, Dim[] dims, String scenarioName, String comment) { - String refData= ""; - Scenario scenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, referenceBuildId, currentBuildId, dims); - String[] timeSeriesLabels= scenario.getTimeSeriesLabels(); - if (timeSeriesLabels.length == 2) { - // we mark the label with a '*' or '†' to indicate that no data was available for the specified builds - if (!timeSeriesLabels[0].equals(referenceBuildId)) { - name= '*' + name; - refData= " (" + timeSeriesLabels[0] + ")"; - } else if (!timeSeriesLabels[1].equals(currentBuildId)) { - name= '†' + name; - refData= " (" + timeSeriesLabels[1] + ")"; - } - } - - for (int i= 0; i < dims.length; i++) { - TimeSeries timeSeries= scenario.getTimeSeries(dims[i]); - int l= timeSeries.getLength(); - if (l >= 1) { - double percent= 0.0; - boolean hasConfidentResult= true; - if (l > 1) { -// hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name); - hasConfidentResult= Utils.hasConfidentResult(timeSeries); - /*if (!rejectNullHypothesis) { - NumberFormat percentFormatter= NumberFormat.getPercentInstance(); - String statisticsComment= "There is not enough evidence to reject the null hypothesis at the " + percentFormatter.format(percentile.inside()) + "level"; - if (comment == null) - comment= statisticsComment; - else - comment+= statisticsComment; - setComment(scenarioName, comment); - } - */ - double ref= timeSeries.getValue(0); - double val= timeSeries.getValue(1); - percent= 100.0 - ((val / ref) * 100.0); - } - if (Math.abs(percent) < 200) { - String n= name + " (" + dims[i].getName() + ")" + refData; - bar.addItem(n, percent,configDescriptor.name+"/"+(scenarioName.replace('#','.').replace(':','_').replace('\\','_'))+".html#"+dims[i].getName(),comment, hasConfidentResult); //$NON-NLS-1$ //$NON-NLS-2$ - } - } - } - //scenario.dump(System.out); - - } - - private void save(String output) { - - //if (bar.getFItems().size()==0) - //return; - - Display display= Display.getDefault(); - - int height= bar.getHeight(); - Image image= new Image(display, GRAPH_WIDTH, height); - - GC gc= new GC(image); - bar.paint(display, GRAPH_WIDTH, height, gc); - gc.dispose(); - - ImageData data = Utils.downSample(image); - ImageLoader il= new ImageLoader(); - il.data= new ImageData[] { data }; - - OutputStream out= null; - try { - out= new BufferedOutputStream(new FileOutputStream(output + ".gif")); //$NON-NLS-1$ - //System.out.println("writing: " + output); - il.save(out, SWT.IMAGE_GIF); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } finally { - image.dispose(); - if (out != null) { - try { - out.close(); - } catch (IOException e1) { - // silently ignored - } - } - } - } - - /* - * Displays bar graph in window - * - private void show(final BarGraph bar) { - Display display= new Display(); - - Shell shell= new Shell(display); - shell.setLayout(new FillLayout()); - - final Canvas canvas= new Canvas(shell, SWT.NO_BACKGROUND); - canvas.addPaintListener(new PaintListener() { - public void paintControl(PaintEvent e) { - Point s= canvas.getSize(); - bar.paint(canvas.getDisplay(), s.x, s.y, e.gc); - } - }); - - shell.open(); - - while (!shell.isDisposed()) - if (!display.readAndDispatch()) - display.sleep(); - - display.dispose(); - } - public BarGraph getBar() { - return bar; +/** + * Create and save fingerprints as image and print their reference in the current stream. + * + * @param performanceResults The performance results used to print the fingerprints + */ +public void print(PerformanceResults performanceResults) { + String baselineBuildName = performanceResults.getBaselineName(); + String buildName = performanceResults.getName(); + + // Compute fingerprint output file name prefix + int referenceUnderscoreIndex = baselineBuildName.indexOf('_'); + String baselinePrefix = baselineBuildName; + if (referenceUnderscoreIndex != -1) { + baselinePrefix = baselineBuildName.substring(0, referenceUnderscoreIndex); } - public String getOutName() { - return outName; + int currentUnderscoreIndex = buildName.indexOf('_'); + if (currentUnderscoreIndex != -1){ + buildName = buildName.substring(0, currentUnderscoreIndex); } - */ + StringBuffer buffer = new StringBuffer("FP_"); + if (this.component != null) { + buffer.append(this.component); + buffer.append('_'); + } + buffer.append(baselinePrefix); + buffer.append('_'); + buffer.append(buildName); + buffer.append('.'); + String filePrefix = buffer.toString(); + + // Create each fingerprint and save it + String[] configNames = performanceResults.getConfigNames(false/* not sorted*/); + String[] configBoxes = performanceResults.getConfigBoxes(false/* not sorted*/); + int length = configNames.length; + for (int c=0; c"; - output = output.concat("" + "" + areas + "\n"); + // Save image file + String fileName = filePrefix + configName ; + File outputFile = new File(this.outputDir, fileName+".gif"); + save(barGraph, outputFile); + + // Print image file reference in stream + String boxName = configBoxes[c]; + if (outputFile.exists()) { + String areas = barGraph.getAreas(); + if (areas == null) areas = ""; + this.stream.print("

"); + this.stream.print(boxName); + this.stream.print("

"); + this.stream.print(""); + this.stream.print(areas); + this.stream.print("\n"); } else { - output = output.concat("

There is no fingerprint for " + componentDescription + "

\n"); + this.stream.print("

There is no fingerprint for "); + this.stream.print(boxName); + this.stream.print("

\n"); } - return output; } } + +/* + * Save the computed bar graph. + */ +private void save(BarGraph barGraph, File outputFile) { + + // Create and paint image + Display display = Display.getDefault(); + int height = barGraph.getHeight(); + Image image = new Image(display, GRAPH_WIDTH, height); + GC gc = new GC(image); + barGraph.paint(display, GRAPH_WIDTH, height, gc); + gc.dispose(); + + // Save image + ImageData data = Utils.downSample(image); + ImageLoader imageLoader = new ImageLoader(); + imageLoader.data = new ImageData[] { data }; + + OutputStream out = null; + try { + out = new BufferedOutputStream(new FileOutputStream(outputFile)); + imageLoader.save(out, SWT.IMAGE_GIF); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + image.dispose(); + if (out != null) { + try { + out.close(); + } catch (IOException e1) { + // silently ignored + } + } + } +} +} Index: src/org/eclipse/test/performance/ui/RawDataTable.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/RawDataTable.java,v retrieving revision 1.7 diff -u -r1.7 RawDataTable.java --- src/org/eclipse/test/performance/ui/RawDataTable.java 7 Apr 2006 21:33:14 -0000 1.7 +++ src/org/eclipse/test/performance/ui/RawDataTable.java 31 Aug 2007 15:14:57 -0000 @@ -10,250 +10,174 @@ *******************************************************************************/ package org.eclipse.test.performance.ui; +import java.io.PrintStream; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Hashtable; -import java.util.Iterator; -import java.util.Set; +import java.util.Collections; +import java.util.List; import org.eclipse.test.internal.performance.data.Dim; -import org.eclipse.test.internal.performance.db.Scenario; -import org.eclipse.test.internal.performance.db.TimeSeries; - +import org.eclipse.test.internal.performance.results.AbstractResults; +import org.eclipse.test.internal.performance.results.BuildResults; +import org.eclipse.test.internal.performance.results.ConfigResults; + +/** + * Class used to fill details file of scenario builds data. + * @see ScenarioData + */ public class RawDataTable { - private Scenario scenario; - private String currentBuild; - private Dim[] dimensions; - private Hashtable buildIDTable; - private Hashtable derivedValues; - private String TOTAL="total"; - private String SAMPLECOUNT="n"; - private String MEAN="mean"; - private String STDDEV="stddev"; - private String CV="cv"; - private ArrayList buildIDPatterns; - - - public RawDataTable(Scenario scenario, Dim[] dimensions, ArrayList buildIDPatterns, String currentBuild) { - this.scenario = scenario; - this.dimensions = dimensions; - this.buildIDPatterns=buildIDPatterns; - this.currentBuild=currentBuild; - buildIDTable = new Hashtable(); - derivedValues=new Hashtable(); - fill(); - } - - public RawDataTable(Scenario scenario, Dim[] dimensions, String buildIDPrefix,String currentBuild) { - buildIDPatterns=new ArrayList(); - buildIDPatterns.add(buildIDPrefix); - this.currentBuild=currentBuild; - this.scenario = scenario; - this.dimensions = dimensions; - buildIDTable = new Hashtable(); - derivedValues=new Hashtable(); - fill(); - } - - private void fill() { - - for (int i = 0; i < dimensions.length; i++) { - TimeSeries ts = scenario.getTimeSeries(dimensions[i]); - double total = 0.0; - int samplesCount=0; - for (int j = 0; j < ts.getLength(); j++) { - String buildID = ts.getLabel(j); - double value = 0.0; - boolean buildIDmatches=false; - Iterator iterator=buildIDPatterns.iterator(); - while (iterator.hasNext()){ - Object tmp=iterator.next(); - if (tmp==null) - continue; - if (buildID.startsWith(tmp.toString())){ - buildIDmatches=true; - break; - } - } - if (!buildIDmatches) - continue; - - Hashtable samples=(Hashtable)buildIDTable.get(buildID); - if (samples==null) - samples=new Hashtable(); - value = ts.getValue(j); - - // store result for build - samples.put(dimensions[i].getName(),new Double(value)); - buildIDTable.put(buildID,samples); - - //keep count of samples added and total value - total+=value; - samplesCount++; - - //quit after current build - if (buildID.equals(currentBuild)) - break; - } - - double mean = total / samplesCount; - double squaredValues = 0.0; - - String[] buildIDs=(String[])buildIDTable.keySet().toArray(new String[buildIDTable.size()]); - for (int j = 0; j < buildIDs.length; j++) { - String buildID = buildIDs[j]; - Hashtable storedValues=(Hashtable)buildIDTable.get(buildID); - double value = ((Double) (storedValues.get(dimensions[i].getName()))).doubleValue(); - double squaredValue = Math.pow(value - mean, 2); - squaredValues += squaredValue; - } + private ConfigResults configResults; + private List buildPrefixes; + private PrintStream stream; + private Dim[] dimensions = AbstractResults.SUPPORTED_DIMS; + private boolean debug = false; + +private RawDataTable(ConfigResults results, PrintStream ps) { + this.configResults = results; + this.stream = ps; +} - double standardDeviation = Math.sqrt((squaredValues / (samplesCount - 1))); - double coefficientOfVariation = Math.round(((standardDeviation) / mean) * 100 * 100) / 100; - - if (coefficientOfVariation>10&&dimensions[i].getName().startsWith("Elapsed")) - System.out.println(scenario.getScenarioName()+": "+" "+coefficientOfVariation); - - //store derived values - Hashtable calculatedValuesForDimension=new Hashtable(); - calculatedValuesForDimension.put(TOTAL,dimensions[i].getDisplayValue(total)); - calculatedValuesForDimension.put(SAMPLECOUNT,new Integer(samplesCount)); - calculatedValuesForDimension.put(MEAN, dimensions[i].getDisplayValue(mean)); - calculatedValuesForDimension.put(STDDEV, dimensions[i].getDisplayValue(standardDeviation)); - calculatedValuesForDimension.put(CV, coefficientOfVariation+"%"); - derivedValues.put(dimensions[i].getName(),calculatedValuesForDimension); - - } - } +public RawDataTable(ConfigResults results, List prefixes, PrintStream ps) { + this(results, ps); + this.buildPrefixes = prefixes; +} +public RawDataTable(ConfigResults results, String baselinePrefix, PrintStream ps) { + this(results, ps); + this.buildPrefixes = new ArrayList(); + this.buildPrefixes.add(baselinePrefix); +} - public String toHtmlString(){ - return ""+htmlSummary()+htmlDetails()+"
\n"; - } - - private String[] sortBuildIDsByDate(String[] buildIDs) { - Hashtable tmp = new Hashtable(); - String[] result = new String[buildIDs.length]; - - for (int i = 0; i < buildIDs.length; i++) { - String date=Utils.getDateFromBuildID(buildIDs[i], true)+""; - if (date.equals("-1")) - date=buildIDs[i]; - tmp.put(date, buildIDs[i]); - } - String[] dates = (String[]) tmp.keySet().toArray(new String[tmp.size()]); - Arrays.sort(dates); - for (int i = 0; i < dates.length; i++) { - result[i] = tmp.get(dates[i]).toString(); - } +/** + * Print all build data to the current stream. + */ +public void print(){ + stream.print(""); + printSummary(); + printDetails(); + stream.println("
"); +} - return result; +/* + * Print table columns headers. + */ +private void printColumnHeaders() { + StringBuffer buffer = new StringBuffer(); + int length = this.dimensions.length; + for (int i=0; i"); + buffer.append(this.dimensions[i].getName()); + buffer.append("
Build ID
"); + stream.print(buildResults.getName()); + stream.print("
Build ID
" + buildID + "
Stats
#BUILDS SAMPLED
MEAN
STD DEV
COEF. VAR
Stats
#BUILDS SAMPLED
MEAN
STD DEV
CV
"+displayValue+"%"+displayValue+"%"+displayValue+"%"+displayValue+"=20) { + stream.print(""); + stream.print(value); + stream.print("%"); +} } Index: META-INF/MANIFEST.MF =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/META-INF/MANIFEST.MF,v retrieving revision 1.8 diff -u -r1.8 MANIFEST.MF --- META-INF/MANIFEST.MF 6 Aug 2007 09:41:21 -0000 1.8 +++ META-INF/MANIFEST.MF 31 Aug 2007 15:14:57 -0000 @@ -11,7 +11,6 @@ org.eclipse.test.performance, org.eclipse.swt, org.apache.derby;resolution:=optional, - org.apache.derby.core;resolution:=optional, Cloudscape;resolution:=optional, org.junit Eclipse-LazyStart: true Index: src/org/eclipse/test/performance/ui/ScenarioData.java =================================================================== RCS file: src/org/eclipse/test/performance/ui/ScenarioData.java diff -N src/org/eclipse/test/performance/ui/ScenarioData.java --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/org/eclipse/test/performance/ui/ScenarioData.java 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,409 @@ +/******************************************************************************* + * Copyright (c) 2004, 2007 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ +package org.eclipse.test.performance.ui; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import junit.framework.AssertionFailedError; + +import org.eclipse.swt.SWT; +import org.eclipse.swt.graphics.Color; +import org.eclipse.swt.graphics.Image; +import org.eclipse.swt.graphics.ImageData; +import org.eclipse.swt.graphics.ImageLoader; +import org.eclipse.swt.widgets.Display; +import org.eclipse.test.internal.performance.data.Dim; +import org.eclipse.test.internal.performance.data.DimensionMessages; +import org.eclipse.test.internal.performance.results.AbstractResults; +import org.eclipse.test.internal.performance.results.BuildResults; +import org.eclipse.test.internal.performance.results.ComponentResults; +import org.eclipse.test.internal.performance.results.ConfigResults; +import org.eclipse.test.internal.performance.results.PerformanceResults; +import org.eclipse.test.internal.performance.results.ScenarioResults; + +/** + * Class used to print scenario all builds data. + */ +public class ScenarioData { + private String baselinePrefix = null; + private List pointsOfInterest; + private List buildIDStreamPatterns; + private File rootDir; + +/** + * Summary of results for a scenario for a given build compared to a + * reference. + * + * @param baselinePrefix The prefix of the baseline build names + * @param pointsOfInterest A list of buildId's to highlight on line graphs + * @param buildIDPatterns + * @param outputDir The directory root where the files are generated + * +*/ +public ScenarioData(String baselinePrefix, List pointsOfInterest, List buildIDPatterns, File outputDir) { + this.baselinePrefix = baselinePrefix; + this.pointsOfInterest = pointsOfInterest; + this.buildIDStreamPatterns = buildIDPatterns; + this.rootDir = outputDir; +} + +/** + * Print the scenario all builds data from the given performance results. + * + * @param performanceResults The needed information to generate scenario data + */ +public void print(PerformanceResults performanceResults) { + String[] configNames = performanceResults.getConfigNames(false/*not sorted*/); + String[] configBoxes = performanceResults.getConfigBoxes(false/*not sorted*/); + int length = configNames.length; + for (int i=0; i 0) { + highlightedPoints.add(builds.get(builds.size()-1)); + } + } else { + highlightedPoints.addAll(builds); + } + } + } + + String scenarioFileName = scenarioResults.getFileName(); + File outFile = new File(outputDir, scenarioFileName + ".html"); + PrintStream stream = null; + try { + stream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outFile))); + } catch (FileNotFoundException e) { + System.err.println("can't create output file" + outFile); //$NON-NLS-1$ + } + if (stream == null) { + stream = System.out; + } + stream.println(Utils.HTML_OPEN); + stream.println(Utils.HTML_DEFAULT_CSS); + + stream.println("" + scenarioResults.getName() + "(" + configBox + ")" + ""); //$NON-NLS-1$ + stream.println("

Scenario: " + scenarioResults.getName() + " (" + configBox + ")


"); //$NON-NLS-1$ //$NON-NLS-2$ + + String failureMessage = Utils.failureMessage(configResults.getCurrentBuildDeviation(), true); + if (failureMessage != null){ + stream.println("
"+failureMessage+"
\n"); + } + + BuildResults currentBuildResults = configResults.getCurrentBuildResults(); + String comment = currentBuildResults.getComment(); + if (comment != null) { + stream.println("

Note:
\n"); + stream.println(comment + "

"); + } + + // Print link to raw data. + String rawDataFile = scenarioFileName+"_raw.html"; + stream.println("

Raw data and Stats

\n"); + stream.println("Click measurement name to view line graph of measured values over builds.

\n"); + + try { + // Print build result table + stream.println(""); //$NON-NLS-1$ + stream.print(""); //$NON-NLS-1$ + Dim[] dimensions = AbstractResults.SUPPORTED_DIMS; + int dimLength = dimensions.length; + for (int d=0; d" + dimName + ""); + } + stream.println("\n"); + + // Write build lines + printTableLine(stream, currentBuildResults); + printTableLine(stream, configResults.getBaselineBuildResults()); + + // Write difference line + printDifferenceLine(stream, configResults); + + // End of table + stream.println("
Build Id
"); + stream.println("*Delta values in red and green indicate degradation > 10% and improvement > 10%,respectively.

"); + stream.println("
\n\n"); + + // print text legend. + stream.println("Black and yellow points plot values measured in integration and last seven nightly builds.
\n" + "Magenta points plot the repeated baseline measurement over time.
\n" + + "Boxed points represent previous releases, milestone builds, current reference and current build.

\n" + + "Hover over any point for build id and value.\n"); + + // print image maps of historical + for (int d=0; d"); + stream.println("
" + dimName + "
"); + stream.println(DimensionMessages.getDescription(dim_id) + "

\n"); + stream.print(""); + stream.print(""); + stream.print(lineGraph.getAreas()); + stream.println(""); + } + stream.println("

"); + stream.println(Utils.HTML_CLOSE); + if (stream != System.out) + stream.close(); + + } catch (AssertionFailedError e) { + e.printStackTrace(); + continue; + } + } +} + +/* + * Print the data for a build results. + */ +private void printTableLine(PrintStream stream, BuildResults buildResults) { + stream.print("
"); + stream.print(buildResults.getName()); + if (buildResults.isBaseline()) stream.print(" (reference)"); + stream.print("
*Delta" + diffDisplayValue + percentage + "" + diffDisplayValue + percentage + "
\n"); + stream.println(""); + stream.print(""); + stream.print(""); + stream.println(""); + stream.println("
Current Stream Test RunsBaseline Test Runs
"); + currentResultsTable.print(); + stream.println(""); + baselineResultsTable.print(); + stream.println("
"); + stream.close(); + } +} + +/* + * Prints a LineGraph object as a gif file. + */ +private void saveGraph(LineGraph p, File outputFile) { + int GRAPH_WIDTH = 600; + int GRAPH_HEIGHT = 200; + Image image = new Image(Display.getDefault(), GRAPH_WIDTH, GRAPH_HEIGHT); + p.paint(image); + + /* Downscale to 8 bit depth palette to save to gif */ + ImageData data = Utils.downSample(image); + ImageLoader il = new ImageLoader(); + il.data = new ImageData[] { data }; + OutputStream out = null; + try { + out = new BufferedOutputStream(new FileOutputStream(outputFile)); + il.save(out, SWT.IMAGE_GIF); + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } finally { + image.dispose(); + if (out != null) { + try { + out.close(); + } catch (IOException e1) { + // silently ignored + } + } + } +} +} #P org.eclipse.test.performance Index: src/org/eclipse/test/internal/performance/data/DimensionMessages.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/data/DimensionMessages.java,v retrieving revision 1.5 diff -u -r1.5 DimensionMessages.java --- src/org/eclipse/test/internal/performance/data/DimensionMessages.java 20 Feb 2006 15:39:47 -0000 1.5 +++ src/org/eclipse/test/internal/performance/data/DimensionMessages.java 31 Aug 2007 15:15:01 -0000 @@ -26,9 +26,13 @@ private DimensionMessages() { // emtpy } - + public static String getString(int id) { - return getString("Dimension." + id); //$NON-NLS-1$ + return getString("dimension." + id); //$NON-NLS-1$ + } + + public static String getDescription(int id) { + return getString("dimension.description." + id); //$NON-NLS-1$ } public static String getString(String key) { Index: src/org/eclipse/test/internal/performance/data/DimensionMessages.properties =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/data/DimensionMessages.properties,v retrieving revision 1.6 diff -u -r1.6 DimensionMessages.properties --- src/org/eclipse/test/internal/performance/data/DimensionMessages.properties 24 Feb 2005 22:19:25 -0000 1.6 +++ src/org/eclipse/test/internal/performance/data/DimensionMessages.properties 31 Aug 2007 15:15:01 -0000 @@ -8,43 +8,87 @@ # Contributors: # IBM Corporation - initial API and implementation ############################################################################### -Dimension.2=System Time -Dimension.3=Used Java Heap -Dimension.4=Working Set -Dimension.7=Committed -Dimension.8=Working Set Peak -Dimension.9=Elapsed Process -Dimension.10=User time -Dimension.11=Kernel time -Dimension.19=Page Faults -Dimension.20=CPU Time -Dimension.22=Commit Limit -Dimension.23=Commit Peak -Dimension.24=Physical Memory -Dimension.25=Physical Available -Dimension.26=System Cache -Dimension.27=Kernel Total -Dimension.28=Kernel Paged -Dimension.29=Kernel Nonpaged -Dimension.30=Page Size -Dimension.31=Handle Count -Dimension.32=Process Count -Dimension.33=Thread Count -Dimension.34=GDI Objects -Dimension.35=USER Objects -Dimension.36=Open Handles -Dimension.37=Commit Total -Dimension.38=Read Count -Dimension.39=Write Count -Dimension.40=Bytes Read -Dimension.41=Bytes Written -Dimension.42=Hard Page Faults -Dimension.43=Soft Page Faults -Dimension.44=Text Size -Dimension.45=Data Size -Dimension.46=Library Size -Dimension.48=Used Memory -Dimension.49=Free Memory -Dimension.50=Buffers Memory -Dimension.51=Free Java Memory -Dimension.52=Invocation Count \ No newline at end of file + +# Dimension Name +dimension.2=System Time +dimension.3=Used Java Heap +dimension.4=Working Set +dimension.7=Committed +dimension.8=Working Set Peak +dimension.9=Elapsed Process +dimension.10=User time +dimension.11=Kernel time +dimension.19=Page Faults +dimension.20=CPU Time +dimension.22=Commit Limit +dimension.23=Commit Peak +dimension.24=Physical Memory +dimension.25=Physical Available +dimension.26=System Cache +dimension.27=Kernel Total +dimension.28=Kernel Paged +dimension.29=Kernel Nonpaged +dimension.30=Page Size +dimension.31=Handle Count +dimension.32=Process Count +dimension.33=Thread Count +dimension.34=GDI Objects +dimension.35=USER Objects +dimension.36=Open Handles +dimension.37=Commit Total +dimension.38=Read Count +dimension.39=Write Count +dimension.40=Bytes Read +dimension.41=Bytes Written +dimension.42=Hard Page Faults +dimension.43=Soft Page Faults +dimension.44=Text Size +dimension.45=Data Size +dimension.46=Library Size +dimension.48=Used Memory +dimension.49=Free Memory +dimension.50=Buffers Memory +dimension.51=Free Java Memory +dimension.52=Invocation Count + +# Dimension Description +dimension.description.2=* no longer measured, same as elapsed time, see PerformanceMonitor * +dimension.description.3=Change in the amount of memory allocated for Java objects. +dimension.description.4=Change in the amount of physical memory used by the process (other data resides in swap space). +dimension.description.7=Change in the amount of allocated memory (both, in physical memory and swap space, can be preallocated for future use). +dimension.description.8=Increase of the maximum working set size, useful with start-up tests. +dimension.description.9=Amount of wall-clock time. +dimension.description.10=User time +dimension.description.11=Amount of time the process ran on the CPU, while the CPU was in kernel mode. +dimension.description.19=Number of memory pages that were loaded from swap space on disk or from memory (i.e., in the latter case, they were not mapped in the process' page table, but already present in memory for some reason). +dimension.description.20=Amount of time the process ran on the CPU. +dimension.description.22=Commit Limit +dimension.description.23=Commit Peak +dimension.description.24=Physical Memory +dimension.description.25=Physical Available +dimension.description.26=System Cache +dimension.description.27=Kernel Total +dimension.description.28=Kernel Paged +dimension.description.29=Kernel Nonpaged +dimension.description.30=Page Size +dimension.description.31=Handle Count +dimension.description.32=Process Count +dimension.description.33=Thread Count +dimension.description.34=Change in the number of GDI (Graphics Device Interface) objects, can be useful for UI tests (particularly start-up tests). +dimension.description.35=USER Objects +dimension.description.36=Open Handles +dimension.description.37=Commit Total +dimension.description.38=Read Count +dimension.description.39=Write Count +dimension.description.40=Bytes Read +dimension.description.41=Bytes Written +dimension.description.42=Number of memory pages that were loaded from swap space on disk. +dimension.description.43=Number of memory pages that were loaded from memory (i.e., they were not mapped in the process' page table, but already present in memory for some reason). +dimension.description.44=Change in the process' code memory size, useful with start-up tests. +dimension.description.45=Change in the process' data and stack memory size. +dimension.description.46=Change in the process' library memory size. +dimension.description.48=Used Memory +dimension.description.49=Free Memory +dimension.description.50=Buffers Memory +dimension.description.51=Free Java Memory +dimension.description.52=Invocation Count Index: src/org/eclipse/test/internal/performance/data/Dim.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/data/Dim.java,v retrieving revision 1.6 diff -u -r1.6 Dim.java --- src/org/eclipse/test/internal/performance/data/Dim.java 20 Feb 2006 15:39:46 -0000 1.6 +++ src/org/eclipse/test/internal/performance/data/Dim.java 31 Aug 2007 15:15:01 -0000 @@ -69,10 +69,14 @@ public boolean largerIsBetter() { return fLargerIsBetter; } - + public String getName() { return DimensionMessages.getString(fId); } + + public String getDescription() { + return DimensionMessages.getString(fId); + } public String toString() { return "Dimension [name=" + getName() + ", " + fUnit + "]"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ Index: .settings/org.eclipse.jdt.core.prefs =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.test.performance/.settings/org.eclipse.jdt.core.prefs,v retrieving revision 1.6 diff -u -r1.6 org.eclipse.jdt.core.prefs --- .settings/org.eclipse.jdt.core.prefs 19 Apr 2006 11:01:35 -0000 1.6 +++ .settings/org.eclipse.jdt.core.prefs 31 Aug 2007 15:15:01 -0000 @@ -1,64 +1,67 @@ -#Wed Apr 19 13:00:25 CEST 2006 -eclipse.preferences.version=1 -org.eclipse.jdt.core.builder.cleanOutputFolder=clean -org.eclipse.jdt.core.builder.duplicateResourceTask=warning -org.eclipse.jdt.core.builder.invalidClasspath=abort -org.eclipse.jdt.core.builder.resourceCopyExclusionFilter=*.launch -org.eclipse.jdt.core.circularClasspath=error -org.eclipse.jdt.core.classpath.exclusionPatterns=enabled -org.eclipse.jdt.core.classpath.multipleOutputLocations=enabled -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=disabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.2 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.4 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate -org.eclipse.jdt.core.compiler.doc.comment.support=enabled -org.eclipse.jdt.core.compiler.maxProblemPerUnit=100 -org.eclipse.jdt.core.compiler.problem.assertIdentifier=warning -org.eclipse.jdt.core.compiler.problem.deprecation=warning -org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled -org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled -org.eclipse.jdt.core.compiler.problem.emptyStatement=warning -org.eclipse.jdt.core.compiler.problem.enumIdentifier=warning -org.eclipse.jdt.core.compiler.problem.fieldHiding=warning -org.eclipse.jdt.core.compiler.problem.finalParameterBound=ignore -org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning -org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=error -org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=error -org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=warning -org.eclipse.jdt.core.compiler.problem.invalidJavadoc=warning -org.eclipse.jdt.core.compiler.problem.invalidJavadocTagsVisibility=private -org.eclipse.jdt.core.compiler.problem.localVariableHiding=warning -org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=error -org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore -org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=enabled -org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=public -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=warning -org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=enabled -org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private -org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning -org.eclipse.jdt.core.compiler.problem.noEffectAssignment=error -org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=error -org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=warning -org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=error -org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=error -org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled -org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=error -org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore -org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=warning -org.eclipse.jdt.core.compiler.problem.unnecessaryElse=warning -org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=error -org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=enabled -org.eclipse.jdt.core.compiler.problem.unusedImport=warning -org.eclipse.jdt.core.compiler.problem.unusedLocal=warning -org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled -org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning -org.eclipse.jdt.core.compiler.source=1.3 -org.eclipse.jdt.core.incompatibleJDKLevel=ignore -org.eclipse.jdt.core.incompleteClasspath=error +#Fri Aug 10 17:30:18 CEST 2007 +eclipse.preferences.version=1 +org.eclipse.jdt.core.builder.cleanOutputFolder=clean +org.eclipse.jdt.core.builder.duplicateResourceTask=warning +org.eclipse.jdt.core.builder.invalidClasspath=abort +org.eclipse.jdt.core.builder.resourceCopyExclusionFilter=*.launch +org.eclipse.jdt.core.circularClasspath=error +org.eclipse.jdt.core.classpath.exclusionPatterns=enabled +org.eclipse.jdt.core.classpath.multipleOutputLocations=enabled +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=disabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.2 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.4 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.doc.comment.support=enabled +org.eclipse.jdt.core.compiler.maxProblemPerUnit=100 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=warning +org.eclipse.jdt.core.compiler.problem.deprecation=warning +org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled +org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled +org.eclipse.jdt.core.compiler.problem.emptyStatement=warning +org.eclipse.jdt.core.compiler.problem.enumIdentifier=warning +org.eclipse.jdt.core.compiler.problem.fieldHiding=warning +org.eclipse.jdt.core.compiler.problem.finalParameterBound=ignore +org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning +org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=error +org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=error +org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=warning +org.eclipse.jdt.core.compiler.problem.invalidJavadoc=warning +org.eclipse.jdt.core.compiler.problem.invalidJavadocTags=enabled +org.eclipse.jdt.core.compiler.problem.invalidJavadocTagsDeprecatedRef=enabled +org.eclipse.jdt.core.compiler.problem.invalidJavadocTagsNotVisibleRef=disabled +org.eclipse.jdt.core.compiler.problem.invalidJavadocTagsVisibility=private +org.eclipse.jdt.core.compiler.problem.localVariableHiding=warning +org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=error +org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore +org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=enabled +org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=public +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=warning +org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=enabled +org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private +org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=error +org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=error +org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=warning +org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=error +org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=error +org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled +org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=error +org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=warning +org.eclipse.jdt.core.compiler.problem.unnecessaryElse=warning +org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=error +org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=enabled +org.eclipse.jdt.core.compiler.problem.unusedImport=warning +org.eclipse.jdt.core.compiler.problem.unusedLocal=warning +org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled +org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning +org.eclipse.jdt.core.compiler.source=1.3 +org.eclipse.jdt.core.incompatibleJDKLevel=ignore +org.eclipse.jdt.core.incompleteClasspath=error Index: src/org/eclipse/test/internal/performance/db/SQL.java =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/db/SQL.java,v retrieving revision 1.34 diff -u -r1.34 SQL.java --- src/org/eclipse/test/internal/performance/db/SQL.java 20 Feb 2006 15:39:45 -0000 1.34 +++ src/org/eclipse/test/internal/performance/db/SQL.java 31 Aug 2007 15:15:01 -0000 @@ -25,7 +25,7 @@ private boolean fCompatibility= false; - private Connection fConnection; + protected Connection fConnection; private PreparedStatement fInsertVariation, fInsertScenario, fInsertSample, fInsertDataPoint, fInsertScalar; private PreparedStatement fQueryComment, fInsertComment, fQueryComment2; @@ -35,7 +35,7 @@ fQuerySummaryEntries; private PreparedStatement fInsertFailure, fQueryFailure; - SQL(Connection con) throws SQLException { + protected SQL(Connection con) throws SQLException { fConnection= con; boolean needsUpgrade= true; @@ -78,7 +78,7 @@ } } - public void dispose() throws SQLException { + protected void dispose() throws SQLException { if (fInsertVariation != null) fInsertVariation.close(); if (fInsertScenario != null) Index: META-INF/MANIFEST.MF =================================================================== RCS file: /cvsroot/eclipse/org.eclipse.test.performance/META-INF/MANIFEST.MF,v retrieving revision 1.7 diff -u -r1.7 MANIFEST.MF --- META-INF/MANIFEST.MF 27 Apr 2007 19:07:34 -0000 1.7 +++ META-INF/MANIFEST.MF 31 Aug 2007 15:15:01 -0000 @@ -11,12 +11,12 @@ org.eclipse.test.internal.performance.data, org.eclipse.test.internal.performance.db, org.eclipse.test.internal.performance.eval, + org.eclipse.test.internal.performance.results, org.eclipse.test.internal.performance.tests, org.eclipse.test.performance Require-Bundle: org.eclipse.core.runtime, org.junit, org.apache.derby;resolution:=optional, - org.apache.derby.core;resolution:=optional, Cloudscape;resolution:=optional Eclipse-LazyStart: true Bundle-ClassPath: . Index: src/org/eclipse/test/internal/performance/results/PerformanceResults.java =================================================================== RCS file: src/org/eclipse/test/internal/performance/results/PerformanceResults.java diff -N src/org/eclipse/test/internal/performance/results/PerformanceResults.java --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/org/eclipse/test/internal/performance/results/PerformanceResults.java 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,244 @@ +/******************************************************************************* + * Copyright (c) 2000, 2006 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ +package org.eclipse.test.internal.performance.results; + +import java.util.*; + + +/** + * Root class to handle performance results. + * + * Usually performance results are built for a current build vs. a baseline build. + * + * This class allow to read all data from releng performance database for given + * configurations and scenario pattern. + * + * Then it provides easy and speedy access to all stored results. + */ +public class PerformanceResults extends AbstractResults { + + final String baselineName; // Name of the baseline build used for comparison + final String baselinePrefix; + private String scenarioPattern; + private List components; + private String[] configNames, sortedConfigNames; + private String[] configBoxes, sortedConfigBoxes; + +public PerformanceResults(String name, String baseline, boolean print) { + super(null, name); + this.baselineName = baseline; + this.baselinePrefix = baseline.substring(0, baseline.lastIndexOf('_')); + this.print = print; +} + +/** + * Returns the name of the baseline used for extracted results + * + * @return The build name of the baseline of null + * if no specific baseline is used for the extracted results. + */ +public String getBaselineName() { + return this.baselineName; +} + +String getBaselinePrefix() { + return this.baselinePrefix; +} + +/** + * Return the list of components concerned by performance results. + * + * @return The list of the components + */ +public List getComponents() { + return components; +} + +/** + * Get the scenarii of a given component. + * + * @param componentName The component name. Should not be null + * @return A list of {@link ScenarioResults scenario results} + */ +public List getComponentScenarii(String componentName) { + ComponentResults componentResults = (ComponentResults) getResults(componentName); + if (componentResults == null) return null; + return Collections.unmodifiableList(componentResults.children); +} + +/** + * Get the scenarii which have a summary for a given component. + * + * @param componentName The component name + * @param config Configuration name + * @return A list of {@link ScenarioResults scenario results} which have a summary + */ +public List getComponentSummaryScenarii(String componentName, String config) { + if (componentName == null) { + int size = size(); + List scenarii = new ArrayList(); + for (int i=0; i< size; i++) { + ComponentResults componentResults = (ComponentResults) this.children.get(i); + scenarii.addAll(componentResults.getSummaryScenarii(true, config)); + } + return scenarii; + } + ComponentResults componentResults = (ComponentResults) getResults(componentName); + return componentResults.getSummaryScenarii(false, config); +} + +/** + * Return the configuration boxes considered for this performance results + * sorted or not depending on the given flag. + * + * @param sort Indicates whether the list must be sorted or not. + * The order is defined by the configuration names, not by the box names + * @return The list of configuration boxes sorted by configuration names + */ +public String[] getConfigBoxes(boolean sort) { + return sort ? this.sortedConfigBoxes : this.configBoxes; +} + +/** + * Return the configuration names considered for this performance results + * sorted or not depending on the given flag. + * + * @param sort Indicates whether the list must be sorted or not + * @return The list of configuration names + */ +public String[] getConfigNames(boolean sort) { + return sort ?this.sortedConfigNames : this.configNames; +} + +String getConfigurationsPattern() { + int length = this.sortedConfigNames == null ? 0 : this.sortedConfigNames.length; + if (length == 0) return null; + String configurations = this.sortedConfigNames[0]; + int refLength = configurations.length(); + for (int i=1; i cannot find a pattern + for (int j=0; j cannot find a pattern + newConfig.append(configurations.substring(0, j)); + } + newConfig.append('_'); + } else if (newConfig != null) { + newConfig.append(c); + } + } + if (newConfig != null) { + configurations = newConfig.toString(); + } + } + return configurations; +} + +PerformanceResults getPerformance() { + return this; +} + +/** + * Get the results of a given scenario. + * + * @param scenarioName The scenario name + * @return The {@link ScenarioResults scenario results} + */ +public ScenarioResults getScenarioResults(String scenarioName) { + ComponentResults componentResults = (ComponentResults) getResults(DB_Results.getComponentNameFromScenario(scenarioName)); + return componentResults == null ? null : (ScenarioResults) componentResults.getResults(scenarioName); +} + +/** + * Read all data from performance database for the given configurations + * and scenario pattern. + * + * @param configs All configs to extract results. If null, + * then all known configurations ({@link #CONFIGS}) are read. + * @param pattern The pattern of the concerned scenarii + */ +public void read(String[][] configs, String pattern) { + + this.scenarioPattern = pattern; + + // Print title + StringBuffer buffer = new StringBuffer("Read performance results until build '"); //$NON-NLS-1$ + buffer.append(this.name); + if (scenarioPattern == null) { + buffer.append("':"); //$NON-NLS-1$ + } else { + buffer.append("' using scenario pattern '"); //$NON-NLS-1$ + buffer.append(scenarioPattern); + buffer.append("':"); //$NON-NLS-1$ + } + println(buffer); + + // Store given configs + if (configs == null) { + int length=CONFIGS.length; + this.configNames = new String[length]; + this.sortedConfigNames = new String[length]; + this.configBoxes = new String[length]; + for (int i=0; i "+(System.currentTimeMillis()-start)+"ms"); //$NON-NLS-1$ //$NON-NLS-2$ + + // Create corresponding children + List allComponents = DB_Results.getAllComponents(); + int size = allComponents.size(); + this.components = new ArrayList(size); + for (int i=0; iconfiguration). + * + * It gives access to results for each build on which this configuration has been run. + * + * @see BuildResults + */ +public class ConfigResults extends AbstractResults { + BuildResults baseline, current; + boolean baselined = false, valid = false; + +public ConfigResults(AbstractResults parent, int id) { + super(parent, id); + this.name = DB_Results.getConfig(id); + this.print = parent.print; +} + +/** + * Returns the baseline build name used to compare results with. + * + * @return The name of the baseline build + * @see #getBaselineBuildResults() + */ +public String getBaselineBuildName() { + return this.baseline.getName(); +} + +/** + * Returns the baseline build results. + *

+ * This build is currently the last reference build which has performance + * + * @return The baseline build results. + * @see BuildResults + */ +public BuildResults getBaselineBuildResults() { + return this.baseline; +} + +/** + * Returns the build results matching a given pattern. + * + * @param buildPattern The pattern of searched builds + * @return The list of the builds which names match the given pattern. + * The list is ordered by build results date. + */ +public List getBuilds(String buildPattern) { + List builds = new ArrayList(); + int size = size(); + for (int i=0; i + * This build is currently the last integration or nightly + * build which has performance results in the database. + * It may differ from the {@link PerformanceResults#getName()}. + * + * @return The current build results. + * @see BuildResults + */ +public BuildResults getCurrentBuildResults() { + return this.current; +} + +/** + * Get all dimension builds statistics for a given list of build prefixes + * and a given dimension. + * + * @param prefixes List of prefixes to filter builds. If null + * then all the builds are taken to compute statistics. + * @param dim_id The id of the dimension on which the statistics must be computed + * @return An array of double built as follows: + * - 0: numbers of values + * - 1: mean of values + * - 2: standard deviation of these values + * - 3: coefficient of variation of these values + */ +public double[] getStatistics(List prefixes, int dim_id) { + int size = size(); + int length = prefixes == null ? 0 : prefixes.size(); + int count = 0; + double mean=0, stddev=0, variation=0; + double[] values = new double[size]; + count = 0; + mean = 0.0; + for (int i=0; itrue if the configuration has results + * for the performance baseline build, false otherwise. + */ +public boolean isBaselined() { + return this.baselined; +} + +/** + * Returns whether the configuration has results for the performance + * current build or not. + * + * @return true if the configuration has results + * for the performance current build, false otherwise. + */ +public boolean isValid() { + return this.valid; +} + +/** + * Returns the 'n' last nightly build names. + * + * @param n Number of last nightly builds to return + * @return Last n nightly build names preceding current. + */ +public List lastNightlyBuildNames(int n) { + List labels = new ArrayList(); + for (int i=size()-2; i>=0; i--) { + BuildResults buildResults = (BuildResults) this.children.get(i); + String buildName = buildResults.getName(); + if (buildName.startsWith("N")) { //$NON-NLS-1$ + labels.add(buildName); + if (labels.size() >= n) break; + } + } + return labels; +} + +/* + * Read scenario results information available in three maps. + */ +void read() { + + // Get performance results builds name + PerformanceResults perfResults = getPerformance(); + String baselineBuildName = perfResults.getBaselineName(); + String currentBuildName = perfResults.getName(); + + // Set baseline and current builds + int size = size(); + for (int i=0; i 0) buffer.append('|'); + buffer.append(baselineName); + } + } + } + return buffer.toString(); +} + +/** + * Return the results of the given configuration. + * + * @param config The configuration name + * @return The {@link ConfigResults results} for the given configuration + * or null if none was found. + */ +public ConfigResults getConfigResults(String config) { + return (ConfigResults) getResults(config); +} + +/** + * Return a name which can be used as a file name to store information + * related to this scenario. This name does not contain the extension. + * + * @return The file name + */ +public String getFileName() { + if (this.fileName == null) { + this.fileName = this.name.replace('#', '.').replace(':', '_').replace('\\', '_'); + } + return this.fileName; +} + +/** + * Returns the scenario label. If no label exist as there's no associated summary, + * then the short name is returned + * + * @return The label of the scenario or it's short name if no summary exists + */ +public String getLabel() { + return this.label; +} + +/** + * Returns the short name of the scenario. Short name is the name scenario + * from which package declaration has been removed. + * + * @return The scenario short name + */ +public String getShortName() { + if (this.shortName == null) { + // Remove class name qualification + int testSeparator = this.name.indexOf('#'); + boolean hasClassName = testSeparator >= 0; + if (!hasClassName) { + testSeparator = this.name.lastIndexOf('.'); + if (testSeparator <= 0) { + return this.shortName = this.name; + } + } + int classSeparator = this.name.substring(0, testSeparator).lastIndexOf('.'); + if (classSeparator < 0) { + return this.shortName = this.name; + } + int length = this.name.length(); + String testName = this.name.substring(classSeparator+1, length); + if (!hasClassName && testName.startsWith("test.")) { // specific case for swt... //$NON-NLS-1$ + testName = testName.substring(5); + } + + // Remove qualification from test name + StringTokenizer tokenizer = new StringTokenizer(testName, " :,", true); //$NON-NLS-1$ + StringBuffer buffer = new StringBuffer(tokenizer.nextToken()); + while (tokenizer.hasMoreTokens()) { + String token = tokenizer.nextToken(); + char fc = token.charAt(0); + while (fc == ' ' || fc == ',' || fc == ':') { + buffer.append(token); // add the separator + token = tokenizer.nextToken(); + fc = token.charAt(0); + } + int last = token.lastIndexOf('.'); + if (last >= 3) { + int first = token .indexOf('.'); + if (first == last) { + buffer.append(token); + } else { + buffer.append(token.substring(last+1)); + } + } else { + buffer.append(token); + } + } + this.shortName = buffer.toString(); + } + return this.shortName; +} + +/** + * Returns whether one of the scenario's config has a summary or not. + * + * @return true if one of the scenario's config has a summary + * false otherwise. + */ +public boolean hasSummary() { + int size = size(); + for (int i=0; itrue if the build has valid results + * false otherwise. + */ +public boolean isValid(String config) { + return getResults(config) != null; +} + +/* + * Read scenario results information from database. + */ +void read() { + + // Get values + print("+ scenario '"+getShortName()+"': "); //$NON-NLS-1$ //$NON-NLS-2$ + long start = System.currentTimeMillis(); + String configPattern = getPerformance().getConfigurationsPattern(); + DB_Results.queryScenarioValues(this, configPattern); + print(" values for "+size()+" configs ("+(System.currentTimeMillis()-start)+"ms), "); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + + // Set baseline and current builds + start = System.currentTimeMillis(); + int size = size(); + for (int i=0; inull. + * + * @param scenarioPattern The pattern of the requested scenarii + * @param buildName TODO + * @return A list of scenario names matching the given pattern + */ +static Map queryAllScenarii(String scenarioPattern, String buildName) { + return getDefault().internalQueryBuildScenarii(scenarioPattern, buildName); +} + +/** + * Get all the failures from DB for a given scenario, configuration + * pattern and builds. + * + * @param scenarioResults The scenario results where to store data + * @param configPattern The configuration pattern concerned by the query + * @param currentBuild The current build to narrow the query + * @param baselineBuild The baseline build to narrow the query + */ +static void queryScenarioFailures(ScenarioResults scenarioResults, String configPattern, BuildResults currentBuild, BuildResults baselineBuild) { + getDefault().internalQueryScenarioFailures(scenarioResults, configPattern, currentBuild, baselineBuild); +} + +/** + * Get all summaries from DB for a given scenario, configuration + * pattern and builds. + * + * @param scenarioResults The scenario results where to store data + * @param configPattern The configuration pattern concerned by the query + * @param currentBuild The current build to narrow the query + * @param baselineBuild The baseline build to narrow the query + */ +static void queryScenarioSummaries(ScenarioResults scenarioResults, String configPattern, BuildResults currentBuild, BuildResults baselineBuild) { + getDefault().internalQueryScenarioSummaries(scenarioResults, configPattern, currentBuild, baselineBuild); +} + +/** + * Query and store all values for given scenario results + * + * @param scenarioResults The scenario results where the values has to be put + * @param configPattern The pattern of the configuration concerned by the query + * +public static void queryScenarioValues(int id, ScenarioResults scenarioResults, String[] configNames) { + getDefault().internalQueryScenarioValues(id, scenarioResults, configNames); +} +*/ +static void queryScenarioValues(ScenarioResults scenarioResults, String configPattern) { + getDefault().internalQueryScenarioValues(scenarioResults, configPattern); +} + +/** + * dbloc= embed in home directory + * dbloc=/tmp/performance embed given location + * dbloc=net://localhost connect to local server + * dbloc=net://www.eclipse.org connect to remove server + */ +private void connect() { + + if (fConnection != null) + return; + + String dbloc = PerformanceTestPlugin.getDBLocation(); + if (dbloc == null) + return; + + String dbname = PerformanceTestPlugin.getDBName(); + String url = null; + java.util.Properties info = new java.util.Properties(); + + if (DEBUG) { + LOG_WRITER.println(); + LOG_WRITER.println("==========================================================="); //$NON-NLS-1$ + LOG_WRITER.println("Database information logged during processing"); //$NON-NLS-1$ + } + + fDBType = DERBY; // assume we are using Derby + try { + if (dbloc.startsWith("net://")) { //$NON-NLS-1$ + // remote + fIsEmbedded = false; + // connect over network + if (DEBUG) + LOG_WRITER.println("Trying to connect over network..."); //$NON-NLS-1$ + Class.forName("com.ibm.db2.jcc.DB2Driver"); //$NON-NLS-1$ + info.put("user", PerformanceTestPlugin.getDBUser()); //$NON-NLS-1$ + info.put("password", PerformanceTestPlugin.getDBPassword()); //$NON-NLS-1$ + info.put("retrieveMessagesFromServerOnGetMessage", "true"); //$NON-NLS-1$ //$NON-NLS-2$ + info.put("create", "true"); //$NON-NLS-1$ //$NON-NLS-2$ + url = dbloc + '/' + dbname; + } else if (dbloc.startsWith("//")) { //$NON-NLS-1$ + // remote + fIsEmbedded = false; + // connect over network + if (DEBUG) + LOG_WRITER.println("Trying to connect over network..."); //$NON-NLS-1$ + Class.forName("org.apache.derby.jdbc.ClientDriver"); //$NON-NLS-1$ + info.put("user", PerformanceTestPlugin.getDBUser()); //$NON-NLS-1$ + info.put("password", PerformanceTestPlugin.getDBPassword()); //$NON-NLS-1$ + info.put("create", "true"); //$NON-NLS-1$ //$NON-NLS-2$ + url = dbloc + '/' + dbname; + } else { + // workaround for Derby issue: + // http://nagoya.apache.org/jira/browse/DERBY-1 + if ("Mac OS X".equals(System.getProperty("os.name"))) //$NON-NLS-1$//$NON-NLS-2$ + System.setProperty("derby.storage.fileSyncTransactionLog", "true"); //$NON-NLS-1$ //$NON-NLS-2$ + + // embedded + fIsEmbedded = true; + try { + Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); //$NON-NLS-1$ + } catch (ClassNotFoundException e) { + Class.forName("com.ihost.cs.jdbc.CloudscapeDriver"); //$NON-NLS-1$ + fDBType = CLOUDSCAPE; + } + if (DEBUG) + LOG_WRITER.println("Loaded embedded " + fDBType); //$NON-NLS-1$ + File f; + if (dbloc.length() == 0) { + String user_home = System.getProperty("user.home"); //$NON-NLS-1$ + if (user_home == null) + return; + f = new File(user_home, fDBType); + } else + f = new File(dbloc); + url = new File(f, dbname).getAbsolutePath(); + info.put("user", PerformanceTestPlugin.getDBUser()); //$NON-NLS-1$ + info.put("password", PerformanceTestPlugin.getDBPassword()); //$NON-NLS-1$ + info.put("create", "true"); //$NON-NLS-1$ //$NON-NLS-2$ + } + try { + fConnection = DriverManager.getConnection("jdbc:" + fDBType + ":" + url, info); //$NON-NLS-1$ //$NON-NLS-2$ + } catch (SQLException e) { + if ("08001".equals(e.getSQLState()) && DERBY.equals(fDBType)) { //$NON-NLS-1$ + if (DEBUG) + LOG_WRITER.println("DriverManager.getConnection failed; retrying for cloudscape"); //$NON-NLS-1$ + // try Cloudscape + fDBType = CLOUDSCAPE; + fConnection = DriverManager.getConnection("jdbc:" + fDBType + ":" + url, info); //$NON-NLS-1$ //$NON-NLS-2$ + } else + throw e; + } + if (DEBUG) + LOG_WRITER.println("connect succeeded!"); //$NON-NLS-1$ + + fConnection.setAutoCommit(false); + fSQL = new SQL_Results(fConnection); + fConnection.commit(); + + } catch (SQLException ex) { + PerformanceTestPlugin.logError(ex.getMessage()); + + } catch (ClassNotFoundException e) { + PerformanceTestPlugin.log(e); + } +} + +private void disconnect() { + if (DEBUG) + LOG_WRITER.println("disconnecting from DB"); //$NON-NLS-1$ + if (fSQL != null) { + try { + fSQL.dispose(); + } catch (SQLException e1) { + PerformanceTestPlugin.log(e1); + } + fSQL = null; + } + if (fConnection != null) { + try { + fConnection.commit(); + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } + try { + fConnection.close(); + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } + fConnection = null; + } + + if (fIsEmbedded) { + try { + DriverManager.getConnection("jdbc:" + fDBType + ":;shutdown=true"); //$NON-NLS-1$ //$NON-NLS-2$ + } catch (SQLException e) { + String message = e.getMessage(); + if (message.indexOf("system shutdown.") < 0) //$NON-NLS-1$ + e.printStackTrace(); + } + } +} + +/* + * Return the index of the build in the list. + */ +private int getBuildId(String build) { + if (BUILDS == null) return -1; + return Arrays.binarySearch(BUILDS, build); +} + +/* + * Return the index of the given configuration in the stored list. + */ +private int getConfigId(String config) { + if (CONFIGS == null) return -1; + return Arrays.binarySearch(CONFIGS, config); +} + +SQL_Results getSQL() { + return fSQL; +} + +/* + * Query all comments from database + */ +private void internalQueryAllComments() { + if (fSQL == null) return; + if (COMMENTS != null) return; + long start = System.currentTimeMillis(); + if (DEBUG) LOG_WRITER.print(" [DB query all comments..."); //$NON-NLS-1$ + ResultSet result = null; + try { + String[] comments = null; + result = fSQL.queryAllComments(); + while (result.next()) { + int commentID = result.getInt(1); + // Ignore kind as there's only one + // int commentKind = result.getInt(2); + String comment = result.getString(3); + if (comments == null) { + comments = new String[commentID+10]; + } else if (commentID >= comments.length) { + int length = comments.length; + System.arraycopy(comments, 0, comments = new String[commentID+10], 0, length); + } + comments[commentID] = comment; + } + COMMENTS = comments; + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } + if (DEBUG) LOG_WRITER.println("done in " + (System.currentTimeMillis() - start) + "ms]"); //$NON-NLS-1$ //$NON-NLS-2$ + } +} + +/* + * Query all variations. This method stores all config and build names. + */ +private void internalQueryAllVariations(String configPattern) { + if (fSQL == null) return; + if (BUILDS != null) return; + long start = System.currentTimeMillis(); + if (DEBUG) { + LOG_WRITER.print(" - DB query all variations for configuration pattern: "+configPattern); //$NON-NLS-1$ + LOG_WRITER.print("..."); //$NON-NLS-1$ + } + ResultSet result = null; + try { + CONFIGS = new String[MAX_CONFIGS]; + BUILDS = null; + BUILDS_LENGTH = 0; + result = fSQL.queryAllVariations(configPattern); + while (result.next()) { + String variation = result.getString(1); // something like "||build=I20070615-1200||config=eclipseperfwin2_R3.3||jvm=sun|" + StringTokenizer tokenizer = new StringTokenizer(variation, "=|"); //$NON-NLS-1$ + tokenizer.nextToken(); // 'build' + String buildName = tokenizer.nextToken(); // 'I20070615-1200' + tokenizer.nextToken(); // 'config' + storeConfig(tokenizer.nextToken()); // 'eclipseperfwin2_R3.3' + tokenizer.nextToken(); // 'jvm' + String vmName = tokenizer.nextToken(); // 'sun' + if (!vmName.equals(SUPPORTED_VMS[0])) { + throw new RuntimeException("Unexpected vm name: "+vmName); //$NON-NLS-1$ + } + storeBuildName(buildName); + } + System.arraycopy(BUILDS, 0, BUILDS = new String[BUILDS_LENGTH], 0, BUILDS_LENGTH); + for (int i=0; i= summaries.length) { + int length = summaries.length; + System.arraycopy(summaries, 0, summaries = new int[variation_id+10], 0, length); + } + summaries[variation_id-1] = global == 1 ? comment_id: -comment_id; + } + } + return summaries; + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } +// this.logWriter.print(", db="+(System.currentTimeMillis()-start)+") "); //$NON-NLS-1$ //$NON-NLS-2$ + if (DEBUG) LOG_WRITER.println("done in " + (System.currentTimeMillis() - start) + "ms]"); //$NON-NLS-1$ //$NON-NLS-2$ + } + return null; +} +*/ +/* +private String[] internalQueryScenarioFailures(int scenarioID, String[] configNames) { + if (fSQL == null) return null; + long start = System.currentTimeMillis(); + if (DEBUG) LOG_WRITER.print(" [DB query all failures from DB for scenario ID: "+scenarioID+"..."); //$NON-NLS-1$ //$NON-NLS-2$ + ResultSet result = null; + try { + String[] failures = null; + int confLength = configNames.length; + for (int config_id=0; config_id= failures.length) { + int length = failures.length; + System.arraycopy(failures, 0, failures = new String[variation_id+10], 0, length); + } + failures[variation_id-1] = message; + } + } + return failures; + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } +// this.logWriter.print(", db="+(System.currentTimeMillis()-start)+") "); //$NON-NLS-1$ //$NON-NLS-2$ + if (DEBUG) LOG_WRITER.println(" -> done in " + (System.currentTimeMillis() - start) + "ms]"); //$NON-NLS-1$ //$NON-NLS-2$ + } + return null; +} +*/ +/* +private Map internalQueryScenariiBuilds(String config) { + if (fSQL == null) return null; + Map scenariiBuilds = (Map) this.allScenariiBuilds.get(config); + if (scenariiBuilds != null) return scenariiBuilds; + long start = System.currentTimeMillis(); + if (DEBUG) this.logWriter.print(" - query all build names from DB for config '"+config+"'..."); //$NON-NLS-1$ //$NON-NLS-2$ + ResultSet result = null; + try { + result = fSQL.queryScenariiBuilds(config); +// this.logWriter.print("(sql="+(System.currentTimeMillis()-start)); //$NON-NLS-1$ +// start = System.currentTimeMillis(); + scenariiBuilds = new HashMap(); + for (int i = 0; result.next(); i++) { + int scenario_id = result.getInt(1); + Integer scenarioID = new Integer(scenario_id); + StringTokenizer variation = new StringTokenizer(result.getString(2), "|="); //$NON-NLS-1$ + variation.nextToken(); // skip first token + List buildNames = (List) scenariiBuilds.get(scenarioID); + if (buildNames == null) { + scenariiBuilds.put(scenarioID, buildNames = new ArrayList()); + } + buildNames.add(variation.nextToken()); + } + this.allScenariiBuilds.put(config, scenariiBuilds); + return scenariiBuilds; + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } +// this.logWriter.print(", db="+(System.currentTimeMillis()-start)+") "); //$NON-NLS-1$ //$NON-NLS-2$ + if (DEBUG) this.logWriter.println("done in " + (System.currentTimeMillis() - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ + } + return null; +} +*/ + +/* +private Map internalQueryScenarioDimValues(String config, int scenarioID, int dimID) { + if (fSQL == null) return null; + long start = System.currentTimeMillis(); + if (DEBUG) { + this.logWriter.println(" - query all data points from DB:"); //$NON-NLS-1$ + this.logWriter.println(" + config = " + config); //$NON-NLS-1$ + this.logWriter.println(" + scenario id = " + scenarioID); //$NON-NLS-1$ + } + ResultSet result = null; + try { + result = fSQL.queryScenarioDimValues(config, scenarioID, dimID); + this.logWriter.print("(sql="+(System.currentTimeMillis()-start)); //$NON-NLS-1$ + start = System.currentTimeMillis(); + if (DEBUG) { + long time = System.currentTimeMillis(); + this.logWriter.println("done in " + (time - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ + start = time; + } + Map dimValues = new HashMap(); + long[] values = null; + String currentBuild = null; + String buildName = null; + while (result.next()) { + int dp_id = result.getInt(1); + int step = result.getInt(2); +// long value = result.getBigDecimal(2).longValue(); + StringTokenizer variation = new StringTokenizer(result.getString(3), "|="); //$NON-NLS-1$ + variation.nextToken(); // skip first token + buildName = variation.nextToken(); + ResultSet rs2 = fSQL.queryScalars(dp_id, dimID); + rs2.next(); + long value = rs2.getBigDecimal(1).longValue(); + rs2.close(); + int idx = InternalPerformanceMeter.AVERAGE - step; + if (!buildName.equals(currentBuild)) { + if (currentBuild != null) dimValues.put(buildName, values); + values = new long[3]; + currentBuild = buildName; + } + values[idx] = value; + } + dimValues.put(buildName, values); + if (DEBUG) this.logWriter.println(" -> " + dimValues.size() + " datapoints created in " + (System.currentTimeMillis() - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + return dimValues; + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } + this.logWriter.print(", db="+(System.currentTimeMillis()-start)+") "); //$NON-NLS-1$ //$NON-NLS-2$ + } + return null; +} +*/ +/* +private Map internalQueryScenarioValues(String config, int scenarioID, Dim[] dims) { + if (fSQL == null) return null; + long start = System.currentTimeMillis(); + if (DEBUG) { + this.logWriter.println(" - query all data points from DB:"); //$NON-NLS-1$ + this.logWriter.println(" + config = " + config); //$NON-NLS-1$ + this.logWriter.println(" + scenario id = " + scenarioID); //$NON-NLS-1$ + } + ResultSet result = null; + try { + result = fSQL.queryScenarioDataPoints(config, scenarioID); +// this.logWriter.print("(sql="+(System.currentTimeMillis()-start)); //$NON-NLS-1$ +// start = System.currentTimeMillis(); + if (DEBUG) { + long time = System.currentTimeMillis(); + this.logWriter.println("done in " + (time - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ + start = time; + } + Map buildsValues = new HashMap(); + String buildName = null; + int count = 0; + int length = dims.length; + while (result.next()) { + int dp_id = result.getInt(1); + int step = result.getInt(2); +// long value = result.getBigDecimal(3).longValue(); + StringTokenizer variation = new StringTokenizer(result.getString(3), "|="); //$NON-NLS-1$ + variation.nextToken(); // skip first token + buildName = variation.nextToken(); + Map dimsValues = (Map) buildsValues.get(buildName); + if (dimsValues == null) { + buildsValues.put(buildName, dimsValues = new HashMap()); + } + int idx = InternalPerformanceMeter.AVERAGE - step; + ResultSet rs2 = fSQL.queryScalars(dp_id); + while (rs2.next()) { + Dim dim = Dim.getDimension(rs2.getInt(1)); + for (int i=0; i " + count + " values read in " + (System.currentTimeMillis() - start) + "ms"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + return buildsValues; + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } +// this.logWriter.print(", db="+(System.currentTimeMillis()-start)+") "); //$NON-NLS-1$ //$NON-NLS-2$ + } + return null; +} +*/ +/* +private void internalQueryScenarioValues(int scenarioId, ScenarioResults scenarioResults, String[] configNames) { + if (fSQL == null) return; + long start = System.currentTimeMillis(); + if (DEBUG) LOG_WRITER.println(" - DB query all data points from DB for scenario: " + scenarioResults.getName()); //$NON-NLS-1$ + internalQueryAllVariations(configNames); // need to read all variations to have all build names + ResultSet result = null; + try { + int length = configNames.length; + int count = 0; + for (int config_id=0; config_id " + count + " values read in " + (System.currentTimeMillis() - start) + "ms]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } + } +} +*/ +/* +private String[] internalQueryScenarioFailures(int scenarioID) { + if (fSQL == null) return null; + long start = System.currentTimeMillis(); + if (DEBUG) LOG_WRITER.print(" - DB query all failures from DB for scenario ID: "+scenarioID+"..."); //$NON-NLS-1$ //$NON-NLS-2$ + ResultSet result = null; + try { + result = fSQL.queryScenarioFailures(scenarioID); +// this.logWriter.print("(sql="+(System.currentTimeMillis()-start)); //$NON-NLS-1$ +// start = System.currentTimeMillis(); + String[] failures = null; + while (result.next()) { + int variationID = result.getInt(1); + String message = result.getString(2); + if (failures == null) { + failures = new String[variationID+10]; + } else if (variationID >= failures.length) { + int length = failures.length; + System.arraycopy(failures, 0, failures = new String[variationID+10], 0, length); + } + failures[variationID] = message; + } + return failures; + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } +// this.logWriter.print(", db="+(System.currentTimeMillis()-start)+") "); //$NON-NLS-1$ //$NON-NLS-2$ + if (DEBUG) LOG_WRITER.println(" -> done in " + (System.currentTimeMillis() - start) + "ms]"); //$NON-NLS-1$ //$NON-NLS-2$ + } + return null; +} +*/ +/* +private int[] internalQueryScenarioSummaries(int scenarioID) { + if (fSQL == null) return null; + long start = System.currentTimeMillis(); + if (DEBUG) LOG_WRITER.print(" - DB query all summaries from DB for scenario ID: "+scenarioID+"..."); //$NON-NLS-1$ //$NON-NLS-2$ + internalQueryAllComments(); + ResultSet result = null; + try { + int[] summaries = null; + result = fSQL.queryScenarioSummaries(scenarioID); +// this.logWriter.print("(sql="+(System.currentTimeMillis()-start)); //$NON-NLS-1$ +// start = System.currentTimeMillis(); + while (result.next()) { + int variationID = result.getInt(1); + int global = result.getShort(2); + int comment_id = result.getInt(3); + if (summaries == null) { + summaries = new int[variationID+10]; + } else if (variationID >= summaries.length) { + int length = summaries.length; + System.arraycopy(summaries, 0, summaries = new int[variationID+10], 0, length); + } + summaries[variationID] = global == 1 ? comment_id: -comment_id; + } + return summaries; + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } +// this.logWriter.print(", db="+(System.currentTimeMillis()-start)+") "); //$NON-NLS-1$ //$NON-NLS-2$ + if (DEBUG) LOG_WRITER.println("done in " + (System.currentTimeMillis() - start) + "ms]"); //$NON-NLS-1$ //$NON-NLS-2$ + } + return null; +} +*/ +private void internalQueryScenarioValues(ScenarioResults scenarioResults, String configPattern) { + if (fSQL == null) return; + long start = System.currentTimeMillis(); + if (DEBUG) LOG_WRITER.print(" - DB query all data points for config pattern: "+configPattern+" for scenario: " + scenarioResults.getShortName()); //$NON-NLS-1$ //$NON-NLS-2$ + internalQueryAllVariations(configPattern); // need to read all variations to have all build names + ResultSet result = null; + try { + int count = 0; + result = fSQL.queryScenarioDataPoints(configPattern, scenarioResults.getId()); + while (result.next()) { + int dp_id = result.getInt(1); + int step = result.getInt(2); + String variation = result.getString(3); // something like "||build=I20070615-1200||config=eclipseperfwin2_R3.3||jvm=sun|" + StringTokenizer tokenizer = new StringTokenizer(variation, "=|"); //$NON-NLS-1$ + tokenizer.nextToken(); // 'build' + String buildName = tokenizer.nextToken(); // 'I20070615-1200' + tokenizer.nextToken(); // 'config' + int config_id = getConfigId(tokenizer.nextToken()); // 'eclipseperfwin2_R3.3' + int build_id = getBuildId(buildName); + ResultSet rs2 = fSQL.queryDimScalars(dp_id); + while (rs2.next()) { + int dim_id = rs2.getInt(1); + long value = rs2.getBigDecimal(2).longValue(); + scenarioResults.setValue(build_id, dim_id, config_id, step, value); + count++; + } + } + if (DEBUG) LOG_WRITER.println(" -> " + count + " values read in " + (System.currentTimeMillis() - start) + "ms]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } + } +} +private void internalQueryScenarioFailures(ScenarioResults scenarioResults, String config, BuildResults currentBuild, BuildResults baselineBuild) { + if (fSQL == null) return; + long start = System.currentTimeMillis(); + if (DEBUG) LOG_WRITER.print(" - DB query all failures for config pattern: "+config+" for scenario : "+scenarioResults.getShortName()+"..."); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + ResultSet result = null; + try { + String currentBuildName = currentBuild.getName(); + String baselineBuildName = baselineBuild.getName(); + result = fSQL.queryScenarioFailures(scenarioResults.getId(), config, currentBuildName, baselineBuildName); + while (result.next()) { + String variation = result.getString(1); + String failure = result.getString(2); + StringTokenizer tokenizer = new StringTokenizer(variation, "=|"); //$NON-NLS-1$ + tokenizer.nextToken(); // 'build' + String buildName = tokenizer.nextToken(); // 'I20070615-1200' +// tokenizer.nextToken(); // 'config' +// String configName = tokenizer.nextToken(); // 'eclipseperfwin2_R3.3' +// int config_id = getConfigId(configName); + if (buildName.equals(currentBuildName)) { + currentBuild.setFailure(failure); + } else if (buildName.equals(baselineBuildName)) { + baselineBuild.setFailure(failure); + } + } + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } +// this.logWriter.print(", db="+(System.currentTimeMillis()-start)+") "); //$NON-NLS-1$ //$NON-NLS-2$ + if (DEBUG) LOG_WRITER.println(" -> done in " + (System.currentTimeMillis() - start) + "ms]"); //$NON-NLS-1$ //$NON-NLS-2$ + } +} + +private void internalQueryScenarioSummaries(ScenarioResults scenarioResults, String config, BuildResults currentBuild, BuildResults baselineBuild) { + if (fSQL == null) return; + long start = System.currentTimeMillis(); + if (DEBUG) LOG_WRITER.print(" - DB query all summaries for config: "+config+" for scenario: "+scenarioResults.getShortName()+"..."); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + internalQueryAllComments(); + ResultSet result = null; + try { + String currentBuildName = currentBuild.getName(); + String baselineBuildName = baselineBuild.getName(); + result = fSQL.queryScenarioSummaries(scenarioResults.getId(), config, currentBuild.getName(), baselineBuild.getName()); + while (result.next()) { + String variation = result.getString(1); + int summaryKind = result.getShort(2); + int comment_id = result.getInt(3); + StringTokenizer tokenizer = new StringTokenizer(variation, "=|"); //$NON-NLS-1$ + tokenizer.nextToken(); // 'build' + String buildName = tokenizer.nextToken(); // 'I20070615-1200' +// tokenizer.nextToken(); // 'config' +// String configName = tokenizer.nextToken(); // 'eclipseperfwin2_R3.3' +// int config_id = getConfigId(configName); + BuildResults buildResults = null; + if (buildName.equals(currentBuildName)) { + buildResults = currentBuild; + } else if (buildName.equals(baselineBuildName)) { + buildResults = baselineBuild; + } + buildResults.setSummary(summaryKind, COMMENTS[comment_id]); + } + } catch (SQLException e) { + PerformanceTestPlugin.log(e); + } finally { + if (result != null) { + try { + result.close(); + } catch (SQLException e1) { + // ignored + } + } + if (DEBUG) LOG_WRITER.println("done in " + (System.currentTimeMillis() - start) + "ms]"); //$NON-NLS-1$ //$NON-NLS-2$ + } +} + +/* + * Store a component in the dynamic list. The list is sorted alphabetically. + */ +private int storeComponent(String component) { + if (COMPONENTS == null) { + COMPONENTS = new String[SUPPORTED_COMPONENTS.length]; + } + int idx = Arrays.binarySearch(SUPPORTED_COMPONENTS, component); + if (idx < 0) { + throw new RuntimeException("Unexpected component name: "+component); //$NON-NLS-1$ + } + if (COMPONENTS[idx] == null) { + COMPONENTS[idx] = SUPPORTED_COMPONENTS[idx]; + } + return idx; +} + +/* + * Store a build name in the dynamic list. + * The list is sorted alphabetically. + */ +private int storeBuildName(String build) { + if (BUILDS == null) { + BUILDS = new String[1]; + BUILDS[BUILDS_LENGTH++] = build; + return 0; + } + int idx = Arrays.binarySearch(BUILDS, build); + if (idx >= 0) return idx; + int index = -idx-1; + int length = BUILDS.length; + if (BUILDS_LENGTH == length) { + String[] array = new String[length+1]; + if (index > 0) System.arraycopy(BUILDS, 0, array, 0, index); + array[index] = build; + if (index < length) System.arraycopy(BUILDS, index+1, array, 0, length-index); + BUILDS = array; + } else if (index < length) { + System.arraycopy(BUILDS, index, BUILDS, index+1, length-index); + BUILDS[index] = build; + } + BUILDS_LENGTH++; + return index; +} + +/* + * Store a configuration in the dynamic list. + * The list is sorted alphabetically. + */ +private int storeConfig(String config) { + for (int i=0; i + * Currently only {@link InternalDimensions#ELAPSED_PROCESS} + * and {@link InternalDimensions#CPU_TIME}. + */ + public final static Dim[] SUPPORTED_DIMS = { + InternalDimensions.ELAPSED_PROCESS, + InternalDimensions.CPU_TIME + }; + + /** + * The list of possible configurations. + *

+ * Only used if no specific configurations are specified + * (see {@link PerformanceResults#read(String[][], String)}. + */ + public final static String[] CONFIGS = { + "eclipseperflnx3_R3.3", //$NON-NLS-1$ + "eclipseperfwin2_R3.3", //$NON-NLS-1$ + "eclipseperflnx2_R3.3", //$NON-NLS-1$ + "eclipseperfwin1_R3.3", //$NON-NLS-1$ + "eclipseperflnx1_R3.3", //$NON-NLS-1$ + }; + + /** + * The list of possible test boxes. + *

+ * Only used if no specific configurations are specified + * (see {@link PerformanceResults#read(String[][], String)}. + */ + public final static String[] BOXES = { + "RHEL 4.0 Sun 1.4.2_10 (3 GHz 2.5 GB)", //$NON-NLS-1$ + "Win XP Sun 1.4.2_10 (3 GHz 2 GB)", //$NON-NLS-1$ + "RHEL 3.0 Sun 1.4.2_10 (3 GHz 2 GB)", //$NON-NLS-1$ + "Win XP Sun 1.4.2_10 (2 GHz 512 MB)", //$NON-NLS-1$ + "RHEL 3.0 Sun 1.4.2_10 (2 GHz 512 MB)", //$NON-NLS-1$ + }; + + + AbstractResults parent; + int id = -1; + String name; + List children; + private boolean newLine = true; + boolean print = false; + +static Dim getDimension(int id) { + int length = SUPPORTED_DIMS.length; + for (int i=0; itrue if the name are equals, + * false otherwise + * @see java.lang.Comparable#compareTo(java.lang.Object) + */ +public boolean equals(Object obj) { + if (obj instanceof AbstractResults) { + return this.name.equals(((AbstractResults)obj).getName()); + } + return super.equals(obj); +} + +int getId() { + return this.id; +} + +/** + * Returns the name of the results object. + * + * @return The name of the results + */ +public String getName() { + return this.name; +} + +PerformanceResults getPerformance() { + if (this.parent != null) { + return this.parent.getPerformance(); + } + return null; +} + +String getPath() { + String path = this.parent==null || this.parent.parent==null ? "" : this.parent.getPath() + ">"; //$NON-NLS-1$ //$NON-NLS-2$ + return path+this.name; +} + +/** + * Return the children list of the current results. + * + * @return An iterator on the children list + */ +public Iterator getResults() { + return this.children.iterator(); +} + +AbstractResults getResults(String resultName) { + int size = this.children.size(); + for (int i=0; i time spent in '"); //$NON-NLS-1$ + buffer.append(this.name); + buffer.append("' was "); //$NON-NLS-1$ + buffer.append((System.currentTimeMillis()-start)/1000.0); + if (end == null) { + buffer.append('s'); + } else { + buffer.append("s. "); //$NON-NLS-1$ + buffer.append(end.trim()); + } + println(buffer); +} + +void println(String text) { + if (this.print) { + if (this.newLine) printTab(); + System.out.println(text); + this.newLine = true; + } +} + +void println(StringBuffer buffer) { + println(buffer.toString()); +} + +public int size() { + return this.children == null ? 0 : this.children.size(); +} + +public String toString() { + return getPath(); +} + +} Index: src/org/eclipse/test/internal/performance/results/ComponentResults.java =================================================================== RCS file: src/org/eclipse/test/internal/performance/results/ComponentResults.java diff -N src/org/eclipse/test/internal/performance/results/ComponentResults.java --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/org/eclipse/test/internal/performance/results/ComponentResults.java 1 Jan 1970 00:00:00 -0000 @@ -0,0 +1,75 @@ +/******************************************************************************* + * Copyright (c) 2000, 2006 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ +package org.eclipse.test.internal.performance.results; + +import java.util.ArrayList; +import java.util.List; + +/** + * Class to handle performance results of an eclipse component + * (for example 'org.eclipse.jdt.core'). + * + * It gives access to results for each scenario run for this component. + * + * @see ScenarioResults + */ +public class ComponentResults extends AbstractResults { + +public ComponentResults(AbstractResults parent, String name) { + super(parent, name); + this.print = parent.print; +} + +ComponentResults getComponent() { + return this; +} + +/** + * Returns a list of scenario results which have a summary + * + * @param global Indicates whether the summary must be global or not. + * @param config Configuration name + * @return A list of {@link ScenarioResults scenario results} which have a summary + */ +public List getSummaryScenarii(boolean global, String config) { + int size= size(); + List scenarii = new ArrayList(size); + for (int i=0; i= 0)) { + scenarii.add(scenarioResults); + } + } + } + return scenarii; +} + +/* + * Read performance results information available in three maps. + */ +void read(List scenarii) { + println("Component '"+this.name+"':"); //$NON-NLS-1$ //$NON-NLS-2$ + long start = System.currentTimeMillis(); + int size = scenarii.size(); + for (int i=0; i"+this.name+" ("+this.count[dim_id]+" measures)..."); //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$//$NON-NLS-4$ //$NON-NLS-5$ + PerformanceTestPlugin.log(status); + } + */ + for (int i=0; inull if no comment was stored for it. + */ +public String getComment() { + if (this.comment == null) { + return null; + } + return this.comment; +} + +/* + * Return the number of stored values for the given dimension + * (see {@link Dim#getId()}) + */ +long getCount(int dim_id) { + return this.count[getDimIndex(dim_id)]; +} + +/** + * Returns the date of the build which is a part of its name. + * + * @return The date of the build as YYYYMMDD + */ +public String getDate() { + if (this.date == null) { + if (this.baseline) { + int length = this.name.length(); + this.date = this.name.substring(length-12, length); + } else { + char first = this.name.charAt(0); + if (first == 'N' || first == 'I' || first == 'M') { // TODO (frederic) should be buildIdPrefixes... + this.date = this.name.substring(1, 9)+this.name.substring(10, 14); + } else { + SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmm"); //$NON-NLS-1$ + int length = this.name.length() - 12 /* length of date */; + for (int i=0; i<=length; i++) { + try { + String substring = i == 0 ? this.name : this.name.substring(i); + format.parse(substring); + this.date = substring; // if no exception is raised then the substring has a correct date format => store it + break; + } catch(ParseException ex) { + // skip + } + } + } + } + } + return this.date; +} + +/** + * Returns the standard deviation of the given dimension computed + * while running the scenario for the current build. + * + * @param dim_id The id of the dimension (see {@link Dim#getId()}) + * @return The value of the standard deviation + */ +public double getDeviation(int dim_id) { + return this.stddev[getDimIndex(dim_id)]; +} + +/* + * Return the index of the dimension corresponding to the given + * dimension id (see {@link Dim#getId()}) + */ +int getDimIndex(int dim_id) { + if (this.dimensions == null) return -1; + int length = this.dimensions.length; + for (int i=0; inull if the scenario passed. + */ +public String getFailure() { + if (this.failure == null) { + return null; + } + return this.failure; +} + +/** + * Return the value of the performance result stored + * for the given dimension of the current build. + * + * @param dim_id The id of the dimension (see {@link Dim#getId()}) + * @return The value of the performance result + */ +public double getValue(int dim_id) { + return this.average[getDimIndex(dim_id)]; +} + +/** + * Returns whether the build is a baseline build or not. + * + * @return true if the build name starts with the baseline prefix + * (see {@link PerformanceResults#getBaselinePrefix()} or false + * otherwise. + */ +public boolean isBaseline() { + return baseline; +} + +/** + * Returns whether the build has a summary or not. Note that the summary + * may be global or not. + * + * @return true if the summary kind is equals to 0 or 1 + * false otherwise. + */ +public boolean hasSummary() { + return this.summaryKind >= 0; +} +/** + * Returns whether the build has a global summary or not. + * + * @return true if the summary kind is equals to 1 + * false otherwise. + */ +public boolean hasGlobalSummary() { + return this.summaryKind == 1; +} + +/* + * Returns a given pattern match the build name or not. + */ +boolean match(String pattern) { + if (pattern.equals("*")) return true; //$NON-NLS-1$ + if (pattern.indexOf('*') < 0 && pattern.indexOf('?') < 0) { + pattern += "*"; //$NON-NLS-1$ + } + StringTokenizer tokenizer = new StringTokenizer(pattern, "*?", true); //$NON-NLS-1$ + int start = 0; + String previous = ""; //$NON-NLS-1$ + while (tokenizer.hasMoreTokens()) { + String token = tokenizer.nextToken(); + if (!token.equals("*") && !token.equals("?")) { //$NON-NLS-1$ //$NON-NLS-2$ + if (previous.equals("*")) { //$NON-NLS-1$ + int idx = this.name.substring(start).indexOf(token); + if (idx < 0) return false; + start += idx; + } else { + if (previous.equals("?")) start++; //$NON-NLS-1$ + if (!name.substring(start).startsWith(token)) return false; + } + start += token.length(); + } + previous = token; + } + if (previous.equals("*")) { //$NON-NLS-1$ + return true; + } else if (previous.equals("?")) { //$NON-NLS-1$ + return this.name.length() == start; + } + return this.name.endsWith(previous); +} + +/* + * Set the build summary and its associated comment. + */ +void setSummary(int kind, String comment) { + this.comment = comment; + this.summaryKind = kind; +} + +/* + * Set the build failure. + */ +void setFailure(String failure) { + this.failure = failure; +} + +/* + * Set the build value from database information. + */ +void setValue(int dim_id, int step, long value) { + int length = SUPPORTED_DIMS.length; + Dim dimension = getDimension(dim_id); + int idx = 0; + if (this.dimensions == null){ + this.dimensions = new Dim[length]; + this.average = new double[length]; + this.stddev = new double[length]; + this.count = new long[length]; + this.dimensions[0] = dimension; + } else { + length = this.dimensions.length; + for (int i=0; i0) buffer.append(", "); //$NON-NLS-1$ + buffer.append('['); + buffer.append(dimensions[i].getId()); + buffer.append("]="); //$NON-NLS-1$ + buffer.append(average[i]); + buffer.append('/'); + buffer.append(count[i]); + buffer.append('/'); + buffer.append(Math.round(stddev[i]*1000)/1000.0); + } + return buffer.toString(); +} + +}