Attachment 113360 Details for Bug 248265 – Proposed patch

[patch] Proposed patch

v00.txt (text/plain), 14.96 KB, created by Frederic Fusier

on 2008-09-24 09:57:16 EDT

(hide)

Description:

Filename:

MIME Type:

Creator: Frederic Fusier

Created: 2008-09-24 09:57:16 EDT

Size: 14.96 KB

patch

obsolete

>### Eclipse Workspace Patch 1.0
>#P org.eclipse.test.performance
>Index: src/org/eclipse/test/internal/performance/results/ConfigResults.java
>===================================================================
>RCS file: /cvsroot/eclipse/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/results/ConfigResults.java,v
>retrieving revision 1.3
>diff -u -r1.3 ConfigResults.java
>--- src/org/eclipse/test/internal/performance/results/ConfigResults.java	23 Sep 2008 11:42:25 -0000	1.3
>+++ src/org/eclipse/test/internal/performance/results/ConfigResults.java	24 Sep 2008 13:53:59 -0000
>@@ -29,6 +29,7 @@
> public class ConfigResults extends AbstractResults {
> 	BuildResults baseline, current;
> 	boolean baselined = false, valid = false;
>+	double delta, error;
> 
> public ConfigResults(AbstractResults parent, int id) {
> 	super(parent, id);
>@@ -118,26 +119,11 @@
>  * @return an array of double. First number is the deviation itself and
>  * 	the second is the standard error.
>  */
>-public double[] getCurrentBuildDeviation() {
>-	if (this.baseline == null || this.current == null) initialize();
>-	int dim_id = SUPPORTED_DIMS[0].getId();
>-	double baselineValue = this.baseline.getValue(dim_id);
>-	double currentValue = this.current.getValue(dim_id);
>-	double deviation = (currentValue - baselineValue) / baselineValue;
>-	if (Double.isNaN(deviation)) {
>-		return new double[] { Double.NaN, Double.NaN };
>-	}
>-	long baselineCount = this.baseline.getCount(dim_id);
>-	long currentCount = this.current.getCount(dim_id);
>-	if (baselineCount == 1 || currentCount == 1) {
>-		return new double[] { deviation, Double.NaN };
>-	}
>-	double baselineError = this.baseline.getError(dim_id);
>-	double currentError = this.current.getError(dim_id);
>-	double stderr = Double.isNaN(baselineError)
>-			? currentError / baselineValue
>-			: Math.sqrt(baselineError*baselineError + currentError*currentError) / baselineValue;
>-	return new double[] { deviation, stderr };
>+public double[] getCurrentBuildDeltaInfo() {
>+	if (this.baseline == null || this.current == null) {
>+		initialize();
>+	}
>+	return new double[] { this.delta, this.error };
> }
> 
> /**
>@@ -244,6 +230,43 @@
> 	if (this.current == null) {
> 		this.current = (BuildResults) this.children.get(size()-1);
> 	}
>+
>+	// Set delta between current vs. baseline and the corresponding error
>+	int dim_id = DEFAULT_DIM.getId();
>+	double baselineValue = this.baseline.getValue(dim_id);
>+	double currentValue = this.current.getValue(dim_id);
>+	this.delta = (currentValue - baselineValue) / baselineValue;
>+	if (Double.isNaN(this.delta)) {
>+		this.error = Double.NaN;
>+	} else {
>+		long baselineCount = this.baseline.getCount(dim_id);
>+		long currentCount = this.current.getCount(dim_id);
>+		if (baselineCount == 1 || currentCount == 1) {
>+			this.error = Double.NaN;
>+		} else {
>+			double baselineError = this.baseline.getError(dim_id);
>+			double currentError = this.current.getError(dim_id);
>+			this.error = Double.isNaN(baselineError)
>+					? currentError / baselineValue
>+					: Math.sqrt(baselineError*baselineError + currentError*currentError) / baselineValue;
>+		}
>+	}
>+
>+	// Set the failure on the current build if necessary
>+	int failure_threshold = getPerformance().failure_threshold;
>+	if (this.delta >= (failure_threshold/100.0)) {
>+		StringBuffer buffer = new StringBuffer("Performance criteria not met when compared to '"); //$NON-NLS-1$
>+		buffer.append(this.baseline.getName());
>+		buffer.append("': "); //$NON-NLS-1$
>+		buffer.append(DEFAULT_DIM.getName());
>+		buffer.append("= "); //$NON-NLS-1$
>+		buffer.append(timeString((long)this.current.getValue()));
>+		buffer.append(" is not within [0%, "); //$NON-NLS-1$
>+		buffer.append(100+failure_threshold);
>+		buffer.append("'%] of "); //$NON-NLS-1$
>+		buffer.append(timeString((long)this.baseline.getValue()));
>+		this.current.setFailure(buffer.toString());
>+	}
> }
> 
> /**
>Index: src/org/eclipse/test/internal/performance/results/AbstractResults.java
>===================================================================
>RCS file: /cvsroot/eclipse/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/results/AbstractResults.java,v
>retrieving revision 1.6
>diff -u -r1.6 AbstractResults.java
>--- src/org/eclipse/test/internal/performance/results/AbstractResults.java	24 Sep 2008 10:00:25 -0000	1.6
>+++ src/org/eclipse/test/internal/performance/results/AbstractResults.java	24 Sep 2008 13:53:59 -0000
>@@ -51,18 +51,19 @@
> 		InternalDimensions.CPU_TIME
> 	};
> 
>+	static final int DEFAULT_DIM_INDEX = 0;
> 	/**
> 	 * The default dimension used to display results (typically in fingerprints).
> 	 * <p>
> 	 * Currently {@link InternalDimensions#ELAPSED_PROCESS}
> 	 */
>-	public static final Dim DEFAULT_DIM = SUPPORTED_DIMS[0];
>+	public static final Dim DEFAULT_DIM = SUPPORTED_DIMS[DEFAULT_DIM_INDEX];
> 
> 	/**
> 	 * The list of possible configurations.
> 	 * <p>
> 	 * Only used if no specific configurations are specified
>-	 * (see {@link PerformanceResults#read(String[][], String)}.
>+	 * (see {@link PerformanceResults#read(File)}.
> 	 */
> 	public final static String[] CONFIGS;
> 
>@@ -70,7 +71,7 @@
> 	 * The list of possible test boxes.
> 	 * <p>
> 	 * Only used if no specific configurations are specified
>-	 * (see {@link PerformanceResults#read(String[][], String)}.
>+	 * (see {@link PerformanceResults#read(File)}.
> 	 */
> 	public final static String[] BOXES = {
> 		"RHEL 4.0 Sun 1.4.2_10 (3 GHz 2.5 GB)", //$NON-NLS-1$
>Index: src/org/eclipse/test/internal/performance/results/PerformanceResults.java
>===================================================================
>RCS file: /cvsroot/eclipse/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/results/PerformanceResults.java,v
>retrieving revision 1.1
>diff -u -r1.1 PerformanceResults.java
>--- src/org/eclipse/test/internal/performance/results/PerformanceResults.java	19 Oct 2007 14:19:07 -0000	1.1
>+++ src/org/eclipse/test/internal/performance/results/PerformanceResults.java	24 Sep 2008 13:53:59 -0000
>@@ -34,6 +34,10 @@
> 	private String[] configBoxes, sortedConfigBoxes;
> 	private String configPattern;
> 
>+	// Failure threshold
>+	public static final int DEFAULT_FAILURE_THRESHOLD = 10;
>+	int failure_threshold = DEFAULT_FAILURE_THRESHOLD;
>+
> public PerformanceResults(String name, String baseline, boolean print) {
> 	super(null, name);
> 	this.baselineName = baseline;
>@@ -189,12 +193,11 @@
>  * Read all data from performance database for the given configurations
>  * and scenario pattern.
>  * 
>- * @param configs All configs to extract results. If <code>null</code>,
>- * 	then all known configurations ({@link #CONFIGS})  are read.
>- * @param pattern The pattern of the concerned scenarios
>+ * @param dataDir The directory where data will be stored locally
>+ * 	if <code>null</code>, then storage will be performed
>  */
>-public void read(String[][] configs, String pattern) {
>-	read(configs, pattern, null);
>+public void read(File dataDir) {
>+	read(null, null, dataDir, DEFAULT_FAILURE_THRESHOLD);
> }
> 
> /**
>@@ -206,10 +209,13 @@
>  * @param pattern The pattern of the concerned scenarios
>  * @param dataDir The directory where data will be stored locally
>  * 	if <code>null</code>, then storage will be performed
>+ * @param threshold The failure percentage threshold over which a build result
>+ * 	value compared to the baseline is considered as failing.
>  */
>-public void read(String[][] configs, String pattern, File dataDir) {
>+public void read(String[][] configs, String pattern, File dataDir, int threshold) {
> 
> 	this.scenarioPattern = pattern;
>+	this.failure_threshold = threshold;
> 
> 	// Print title
> 	StringBuffer buffer = new StringBuffer("Read performance results until build '"); //$NON-NLS-1$
>Index: src/org/eclipse/test/internal/performance/results/BuildResults.java
>===================================================================
>RCS file: /cvsroot/eclipse/org.eclipse.test.performance/src/org/eclipse/test/internal/performance/results/BuildResults.java,v
>retrieving revision 1.7
>diff -u -r1.7 BuildResults.java
>--- src/org/eclipse/test/internal/performance/results/BuildResults.java	23 Sep 2008 11:42:25 -0000	1.7
>+++ src/org/eclipse/test/internal/performance/results/BuildResults.java	24 Sep 2008 13:53:59 -0000
>@@ -27,17 +27,20 @@
> 
> 	// Build information
> 	String date;
>-	boolean baseline;
>-	String failure;
> 	String comment;
> 	int summaryKind = -1;
> 
>-	// Dimensions informations
>+	// Dimensions information
> 	Dim[] dimensions;
> 	double[] average, stddev;
> 	long[] count;
> 	double[][] values;
>-	BuildResults(AbstractResults parent) {
>+
>+	// Comparison information
>+	boolean baseline;
>+	String failure;
>+
>+BuildResults(AbstractResults parent) {
> 	super(parent, -1);
> }
> 
>@@ -195,9 +198,7 @@
>  * @return The failure message or <code>null</null> if the scenario passed.
>  */
> public String getFailure() {
>-	if (this.failure == null) {
>-		return null;
>-	}
>+	if (this.comment != null) return this.comment;
> 	return this.failure;
> }
> 
>@@ -213,6 +214,16 @@
> }
> 
> /**
>+ * Return the value of the performance result stored
>+ * for the default dimension of the current build.
>+ * 
>+ * @return The value of the performance result
>+ */
>+public double getValue() {
>+	return this.average[DEFAULT_DIM_INDEX];
>+}
>+
>+/**
>  * Returns whether the build is a baseline build or not.
>  * 
>  * @return <code>true</code> if the build name starts with the baseline prefix
>#P org.eclipse.test.performance.ui
>Index: src/org/eclipse/test/performance/ui/Main.java
>===================================================================
>RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/Main.java,v
>retrieving revision 1.50
>diff -u -r1.50 Main.java
>--- src/org/eclipse/test/performance/ui/Main.java	23 Sep 2008 11:42:24 -0000	1.50
>+++ src/org/eclipse/test/performance/ui/Main.java	24 Sep 2008 13:54:01 -0000
>@@ -172,6 +172,13 @@
>  */
> private boolean use_php = true; // PerformanceTestPlugin.getDBLocation().startsWith("net://");
> 
>+/**
>+ * Tells what should be the failure percentage threshold.
>+ * <p>
>+ * Default is 10%.
>+ */
>+private int failure_threshold = 10; // PerformanceTestPlugin.getDBLocation().startsWith("net://");
>+
> /*
>  * Parse the command arguments and create corresponding performance
>  * results object.
>@@ -417,6 +424,23 @@
> 			i++;
> 			continue;
> 		}
>+		if (arg.equals("-failure.threshold")) {
>+			String value = args[i + 1];
>+			try {
>+				this.failure_threshold = Integer.parseInt(value);
>+				if (this.failure_threshold < 0) {
>+					System.out.println("Value for "+arg+" parameter must be positive.");
>+					printUsage();
>+				}
>+			}
>+			catch (NumberFormatException nfe) {
>+				System.out.println("Invalid value for "+arg+" parameter");
>+				printUsage();
>+			}
>+			buffer.append("	").append(arg).append(" = ").append(value).append('\n');
>+			i++;
>+			continue;
>+		}
> 		i++;
> 	}
> 	if (this.print) System.out.println(buffer.toString());
>@@ -705,7 +729,13 @@
> 		"	Optional.  Generates table of scenario reference and current data with line graphs.\n\n" +
> 
> 		"[-print]\n" +
>-		"	Optional.  Display output in the console while generating.\n"
>+		"	Optional.  Display output in the console while generating.\n" +
>+
>+		"[-nophp]\n" +
>+		"	Optional.  Generate files for non-php server.\n" +
>+
>+		"[-failure.threshold]\n" +
>+		"	Optional.  Set the failure percentage threshold (default is 10%).\n"
> 	);
> 
> 	System.exit(1);
>@@ -728,7 +758,7 @@
> 
> 	// Parse arguments and read DB info
> 	PerformanceResults performanceResults = parse(context.getArguments().get("application.args"));
>-	performanceResults.read(this.configDescriptors, this.scenarioPattern, this.dataDir);
>+	performanceResults.read(this.configDescriptors, this.scenarioPattern, this.dataDir, this.failure_threshold);
> 
> 	// Print whole scenarios summary
> 	if (this.print) System.out.println();
>Index: src/org/eclipse/test/performance/ui/ScenarioStatusTable.java
>===================================================================
>RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/ScenarioStatusTable.java,v
>retrieving revision 1.35
>diff -u -r1.35 ScenarioStatusTable.java
>--- src/org/eclipse/test/performance/ui/ScenarioStatusTable.java	5 Nov 2007 14:53:26 -0000	1.35
>+++ src/org/eclipse/test/performance/ui/ScenarioStatusTable.java	24 Sep 2008 13:54:01 -0000
>@@ -128,7 +128,7 @@
> 	}
> 	BuildResults currentBuildResults = configResults.getCurrentBuildResults();
> 	String failure = currentBuildResults.getFailure();
>-	double[] deviation = configResults.getCurrentBuildDeviation();
>+	double[] deviation = configResults.getCurrentBuildDeltaInfo();
> 	int confidence = Utils.confidenceLevel(deviation);
> 	boolean hasFailure = failure != null;
> 	String comment = currentBuildResults.getComment();
>Index: src/org/eclipse/test/performance/ui/FingerPrint.java
>===================================================================
>RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/FingerPrint.java,v
>retrieving revision 1.27
>diff -u -r1.27 FingerPrint.java
>--- src/org/eclipse/test/performance/ui/FingerPrint.java	24 Sep 2008 10:00:27 -0000	1.27
>+++ src/org/eclipse/test/performance/ui/FingerPrint.java	24 Sep 2008 13:54:01 -0000
>@@ -94,7 +94,7 @@
> 			ScenarioResults scenarioResults = (ScenarioResults) scenarios.get(i);
> 			ConfigResults configResults = scenarioResults.getConfigResults(configName);
> 			if (configResults == null || !configResults.isValid()) continue;
>-			double[] results = configResults.getCurrentBuildDeviation();
>+			double[] results = configResults.getCurrentBuildDeltaInfo();
> 			double percent = -results[0] * 100.0;
> 			if (results != null && Math.abs(percent) < 200) {
> 				String name = scenarioResults.getLabel() + " (" + defaultDimName + ")";
>Index: src/org/eclipse/test/performance/ui/ScenarioData.java
>===================================================================
>RCS file: /cvsroot/eclipse/org.eclipse.releng.basebuilder/plugins/org.eclipse.test.performance.ui/src/org/eclipse/test/performance/ui/ScenarioData.java,v
>retrieving revision 1.6
>diff -u -r1.6 ScenarioData.java
>--- src/org/eclipse/test/performance/ui/ScenarioData.java	24 Sep 2008 10:00:27 -0000	1.6
>+++ src/org/eclipse/test/performance/ui/ScenarioData.java	24 Sep 2008 13:54:01 -0000
>@@ -217,7 +217,7 @@
> 		stream.println("<title>" + scenarioResults.getName() + "(" + configBox + ")" + "</title></head>"); //$NON-NLS-1$
> 		stream.println("<h4>Scenario: " + scenarioResults.getName() + " (" + configBox + ")</h4><br>"); //$NON-NLS-1$ //$NON-NLS-2$
> 
>-		String failureMessage = Utils.failureMessage(configResults.getCurrentBuildDeviation(), true);
>+		String failureMessage = Utils.failureMessage(configResults.getCurrentBuildDeltaInfo(), true);
>  		if (failureMessage != null){
>    			stream.println("<table><tr><td><b>"+failureMessage+"</td></tr></table>\n");
>  		}

Actions: View | Diff

Attachments on bug 248265: 113360