Attachment #113360 for bug #248265

View | Details | Raw Unified | Return to bug 248265
Collapse All | Expand All




public class ConfigResults extends AbstractResults {
	BuildResults baseline, current;
	boolean baselined = false, valid = false;
	double delta, error;

public ConfigResults(AbstractResults parent, int id) {
	super(parent, id);

 * @return an array of double. First number is the deviation itself and
 * 	the second is the standard error.
 */
public double[] getCurrentBuildDeltaInfo() {
	if (this.baseline == null || this.current == null) {
		initialize();
	}
	return new double[] { this.delta, this.error };
	double deviation = (currentValue - baselineValue) / baselineValue;
	if (Double.isNaN(deviation)) {
		return new double[] { Double.NaN, Double.NaN };
	}
	long baselineCount = this.baseline.getCount(dim_id);
	long currentCount = this.current.getCount(dim_id);
	if (baselineCount == 1 || currentCount == 1) {
		return new double[] { deviation, Double.NaN };
	}
	double baselineError = this.baseline.getError(dim_id);
	double currentError = this.current.getError(dim_id);
	double stderr = Double.isNaN(baselineError)
			? currentError / baselineValue
			: Math.sqrt(baselineError*baselineError + currentError*currentError) / baselineValue;
	return new double[] { deviation, stderr };
}

/**

	if (this.current == null) {
		this.current = (BuildResults) this.children.get(size()-1);
	}

	// Set delta between current vs. baseline and the corresponding error
	int dim_id = DEFAULT_DIM.getId();
	double baselineValue = this.baseline.getValue(dim_id);
	double currentValue = this.current.getValue(dim_id);
	this.delta = (currentValue - baselineValue) / baselineValue;
	if (Double.isNaN(this.delta)) {
		this.error = Double.NaN;
	} else {
		long baselineCount = this.baseline.getCount(dim_id);
		long currentCount = this.current.getCount(dim_id);
		if (baselineCount == 1 || currentCount == 1) {
			this.error = Double.NaN;
		} else {
			double baselineError = this.baseline.getError(dim_id);
			double currentError = this.current.getError(dim_id);
			this.error = Double.isNaN(baselineError)
					? currentError / baselineValue
					: Math.sqrt(baselineError*baselineError + currentError*currentError) / baselineValue;
		}
	}

	// Set the failure on the current build if necessary
	int failure_threshold = getPerformance().failure_threshold;
	if (this.delta >= (failure_threshold/100.0)) {
		StringBuffer buffer = new StringBuffer("Performance criteria not met when compared to '"); //$NON-NLS-1$
		buffer.append(this.baseline.getName());
		buffer.append("': "); //$NON-NLS-1$
		buffer.append(DEFAULT_DIM.getName());
		buffer.append("= "); //$NON-NLS-1$
		buffer.append(timeString((long)this.current.getValue()));
		buffer.append(" is not within [0%, "); //$NON-NLS-1$
		buffer.append(100+failure_threshold);
		buffer.append("'%] of "); //$NON-NLS-1$
		buffer.append(timeString((long)this.baseline.getValue()));
		this.current.setFailure(buffer.toString());
	}
}

/**




		InternalDimensions.CPU_TIME
	};

	static final int DEFAULT_DIM_INDEX = 0;
	/**
	 * The default dimension used to display results (typically in fingerprints).
	 * <p>
	 * Currently {@link InternalDimensions#ELAPSED_PROCESS}
	 */
	public static final Dim DEFAULT_DIM = SUPPORTED_DIMS[DEFAULT_DIM_INDEX];

	/**
	 * The list of possible configurations.
	 * <p>
	 * Only used if no specific configurations are specified
	 * (see {@link PerformanceResults#read(File)}.
	 */
	public final static String[] CONFIGS;


	 * The list of possible test boxes.
	 * <p>
	 * Only used if no specific configurations are specified
	 * (see {@link PerformanceResults#read(File)}.
	 */
	public final static String[] BOXES = {
		"RHEL 4.0 Sun 1.4.2_10 (3 GHz 2.5 GB)", //$NON-NLS-1$

Lines 34-39 Link Here

(-)src/org/eclipse/test/internal/performance/results/PerformanceResults.java (-6 / +12 lines)
34	private String[] configBoxes, sortedConfigBoxes;	34	private String[] configBoxes, sortedConfigBoxes;
35	private String configPattern;	35	private String configPattern;
36		36
		37	// Failure threshold
		38	public static final int DEFAULT_FAILURE_THRESHOLD = 10;
		39	int failure_threshold = DEFAULT_FAILURE_THRESHOLD;
		40
37	public PerformanceResults(String name, String baseline, boolean print) {	41	public PerformanceResults(String name, String baseline, boolean print) {
38	super(null, name);	42	super(null, name);
39	this.baselineName = baseline;	43	this.baselineName = baseline;
Lines 189-200 Link Here
189	* Read all data from performance database for the given configurations	193	* Read all data from performance database for the given configurations
190	* and scenario pattern.	194	* and scenario pattern.
191	*	195	*
192	* @param configs All configs to extract results. If <code>null</code>,	196	* @param dataDir The directory where data will be stored locally
193	* then all known configurations ({@link #CONFIGS}) are read.	197	* if <code>null</code>, then storage will be performed
194	* @param pattern The pattern of the concerned scenarios
195	*/	198	*/
196	public void read(String[][] configs, String pattern) {	199	public void read(File dataDir) {
197	read(configs, pattern, null);	200	read(null, null, dataDir, DEFAULT_FAILURE_THRESHOLD);
198	}	201	}
199		202
200	/**	203	/**
Lines 206-215 Link Here
206	* @param pattern The pattern of the concerned scenarios	209	* @param pattern The pattern of the concerned scenarios
207	* @param dataDir The directory where data will be stored locally	210	* @param dataDir The directory where data will be stored locally
208	* if <code>null</code>, then storage will be performed	211	* if <code>null</code>, then storage will be performed
		212	* @param threshold The failure percentage threshold over which a build result
		213	* value compared to the baseline is considered as failing.
209	*/	214	*/
210	public void read(String[][] configs, String pattern, File dataDir) {	215	public void read(String[][] configs, String pattern, File dataDir, int threshold) {
211		216
212	this.scenarioPattern = pattern;	217	this.scenarioPattern = pattern;
		218	this.failure_threshold = threshold;
213		219
214	// Print title	220	// Print title
215	StringBuffer buffer = new StringBuffer("Read performance results until build '"); //$NON-NLS-1$	221	StringBuffer buffer = new StringBuffer("Read performance results until build '"); //$NON-NLS-1$





	// Build information
	String date;
	boolean baseline;
	String failure;
	String comment;
	int summaryKind = -1;

	// Dimensions information
	Dim[] dimensions;
	double[] average, stddev;
	long[] count;
	double[][] values;

	// Comparison information
	boolean baseline;
	String failure;

BuildResults(AbstractResults parent) {
	super(parent, -1);
}


 * @return The failure message or <code>null</null> if the scenario passed.
 */
public String getFailure() {
	if (this.comment != null) return this.comment;
		return null;
	}
	return this.failure;
}


}

/**
 * Return the value of the performance result stored
 * for the default dimension of the current build.
 * 
 * @return The value of the performance result
 */
public double getValue() {
	return this.average[DEFAULT_DIM_INDEX];
}

/**
 * Returns whether the build is a baseline build or not.
 * 
 * @return <code>true</code> if the build name starts with the baseline prefix

Lines 172-177 Link Here

(-)src/org/eclipse/test/performance/ui/Main.java (-2 / +32 lines)
172	*/	172	*/
173	private boolean use_php = true; // PerformanceTestPlugin.getDBLocation().startsWith("net://");	173	private boolean use_php = true; // PerformanceTestPlugin.getDBLocation().startsWith("net://");
174		174
		175	/**
		176	* Tells what should be the failure percentage threshold.
		177	* <p>
		178	* Default is 10%.
		179	*/
		180	private int failure_threshold = 10; // PerformanceTestPlugin.getDBLocation().startsWith("net://");
		181
175	/*	182	/*
176	* Parse the command arguments and create corresponding performance	183	* Parse the command arguments and create corresponding performance
177	* results object.	184	* results object.
Lines 417-422 Link Here
417	i++;	424	i++;
418	continue;	425	continue;
419	}	426	}
		427	if (arg.equals("-failure.threshold")) {
		428	String value = args[i + 1];
		429	try {
		430	this.failure_threshold = Integer.parseInt(value);
		431	if (this.failure_threshold < 0) {
		432	System.out.println("Value for "+arg+" parameter must be positive.");
		433	printUsage();
		434	}
		435	}
		436	catch (NumberFormatException nfe) {
		437	System.out.println("Invalid value for "+arg+" parameter");
		438	printUsage();
		439	}
		440	buffer.append(" ").append(arg).append(" = ").append(value).append('\n');
		441	i++;
		442	continue;
		443	}
420	i++;	444	i++;
421	}	445	}
422	if (this.print) System.out.println(buffer.toString());	446	if (this.print) System.out.println(buffer.toString());
Lines 705-711 Link Here
705	" Optional. Generates table of scenario reference and current data with line graphs.\n\n" +	729	" Optional. Generates table of scenario reference and current data with line graphs.\n\n" +
706		730
707	"[-print]\n" +	731	"[-print]\n" +
708	" Optional. Display output in the console while generating.\n"	732	" Optional. Display output in the console while generating.\n" +
		733
		734	"[-nophp]\n" +
		735	" Optional. Generate files for non-php server.\n" +
		736
		737	"[-failure.threshold]\n" +
		738	" Optional. Set the failure percentage threshold (default is 10%).\n"
709	);	739	);
710		740
711	System.exit(1);	741	System.exit(1);
Lines 728-734 Link Here
728		758
729	// Parse arguments and read DB info	759	// Parse arguments and read DB info
730	PerformanceResults performanceResults = parse(context.getArguments().get("application.args"));	760	PerformanceResults performanceResults = parse(context.getArguments().get("application.args"));
731	performanceResults.read(this.configDescriptors, this.scenarioPattern, this.dataDir);	761	performanceResults.read(this.configDescriptors, this.scenarioPattern, this.dataDir, this.failure_threshold);
732		762
733	// Print whole scenarios summary	763	// Print whole scenarios summary
734	if (this.print) System.out.println();	764	if (this.print) System.out.println();

Lines 128-134 Link Here

(-)src/org/eclipse/test/performance/ui/ScenarioStatusTable.java (-1 / +1 lines)
128	}	128	}
129	BuildResults currentBuildResults = configResults.getCurrentBuildResults();	129	BuildResults currentBuildResults = configResults.getCurrentBuildResults();
130	String failure = currentBuildResults.getFailure();	130	String failure = currentBuildResults.getFailure();
131	double[] deviation = configResults.getCurrentBuildDeviation();	131	double[] deviation = configResults.getCurrentBuildDeltaInfo();
132	int confidence = Utils.confidenceLevel(deviation);	132	int confidence = Utils.confidenceLevel(deviation);
133	boolean hasFailure = failure != null;	133	boolean hasFailure = failure != null;
134	String comment = currentBuildResults.getComment();	134	String comment = currentBuildResults.getComment();

Lines 94-100 Link Here

(-)src/org/eclipse/test/performance/ui/FingerPrint.java (-1 / +1 lines)
94	ScenarioResults scenarioResults = (ScenarioResults) scenarios.get(i);	94	ScenarioResults scenarioResults = (ScenarioResults) scenarios.get(i);
95	ConfigResults configResults = scenarioResults.getConfigResults(configName);	95	ConfigResults configResults = scenarioResults.getConfigResults(configName);
96	if (configResults == null \|\| !configResults.isValid()) continue;	96	if (configResults == null \|\| !configResults.isValid()) continue;
97	double[] results = configResults.getCurrentBuildDeviation();	97	double[] results = configResults.getCurrentBuildDeltaInfo();
98	double percent = -results[0] * 100.0;	98	double percent = -results[0] * 100.0;
99	if (results != null && Math.abs(percent) < 200) {	99	if (results != null && Math.abs(percent) < 200) {
100	String name = scenarioResults.getLabel() + " (" + defaultDimName + ")";	100	String name = scenarioResults.getLabel() + " (" + defaultDimName + ")";

Lines 217-223 Link Here

(-)src/org/eclipse/test/performance/ui/ScenarioData.java (-1 / +1 lines)
217	stream.println("<title>" + scenarioResults.getName() + "(" + configBox + ")" + "</title></head>"); //$NON-NLS-1$	217	stream.println("<title>" + scenarioResults.getName() + "(" + configBox + ")" + "</title></head>"); //$NON-NLS-1$
218	stream.println("<h4>Scenario: " + scenarioResults.getName() + " (" + configBox + ")</h4><br>"); //$NON-NLS-1$ //$NON-NLS-2$	218	stream.println("<h4>Scenario: " + scenarioResults.getName() + " (" + configBox + ")</h4><br>"); //$NON-NLS-1$ //$NON-NLS-2$
219		219
220	String failureMessage = Utils.failureMessage(configResults.getCurrentBuildDeviation(), true);	220	String failureMessage = Utils.failureMessage(configResults.getCurrentBuildDeltaInfo(), true);
221	if (failureMessage != null){	221	if (failureMessage != null){
222	stream.println("<table><tr><td><b>"+failureMessage+"</td></tr></table>\n");	222	stream.println("<table><tr><td><b>"+failureMessage+"</td></tr></table>\n");
223	}	223	}

Return to bug 248265

Lines 29-34 Link Here

(-)src/org/eclipse/test/internal/performance/results/ConfigResults.java (-20 / +43 lines)
29	public class ConfigResults extends AbstractResults {	29	public class ConfigResults extends AbstractResults {
30	BuildResults baseline, current;	30	BuildResults baseline, current;
31	boolean baselined = false, valid = false;	31	boolean baselined = false, valid = false;
		32	double delta, error;
32		33
33	public ConfigResults(AbstractResults parent, int id) {	34	public ConfigResults(AbstractResults parent, int id) {
34	super(parent, id);	35	super(parent, id);
Lines 118-143 Link Here
118	* @return an array of double. First number is the deviation itself and	119	* @return an array of double. First number is the deviation itself and
119	* the second is the standard error.	120	* the second is the standard error.
120	*/	121	*/
121	public double[] getCurrentBuildDeviation() {	122	public double[] getCurrentBuildDeltaInfo() {
122	if (this.baseline == null \|\| this.current == null) initialize();	123	if (this.baseline == null \|\| this.current == null) {
123	int dim_id = SUPPORTED_DIMS[0].getId();	124	initialize();
124	double baselineValue = this.baseline.getValue(dim_id);	125	}
125	double currentValue = this.current.getValue(dim_id);	126	return new double[] { this.delta, this.error };
126	double deviation = (currentValue - baselineValue) / baselineValue;
127	if (Double.isNaN(deviation)) {
128	return new double[] { Double.NaN, Double.NaN };
129	}
130	long baselineCount = this.baseline.getCount(dim_id);
131	long currentCount = this.current.getCount(dim_id);
132	if (baselineCount == 1 \|\| currentCount == 1) {
133	return new double[] { deviation, Double.NaN };
134	}
135	double baselineError = this.baseline.getError(dim_id);
136	double currentError = this.current.getError(dim_id);
137	double stderr = Double.isNaN(baselineError)
138	? currentError / baselineValue
139	: Math.sqrt(baselineErrorbaselineError + currentErrorcurrentError) / baselineValue;
140	return new double[] { deviation, stderr };
141	}	127	}
142		128
143	/**	129	/**
Lines 244-249 Link Here
244	if (this.current == null) {	230	if (this.current == null) {
245	this.current = (BuildResults) this.children.get(size()-1);	231	this.current = (BuildResults) this.children.get(size()-1);
246	}	232	}
		233
		234	// Set delta between current vs. baseline and the corresponding error
		235	int dim_id = DEFAULT_DIM.getId();
		236	double baselineValue = this.baseline.getValue(dim_id);
		237	double currentValue = this.current.getValue(dim_id);
		238	this.delta = (currentValue - baselineValue) / baselineValue;
		239	if (Double.isNaN(this.delta)) {
		240	this.error = Double.NaN;
		241	} else {
		242	long baselineCount = this.baseline.getCount(dim_id);
		243	long currentCount = this.current.getCount(dim_id);
		244	if (baselineCount == 1 \|\| currentCount == 1) {
		245	this.error = Double.NaN;
		246	} else {
		247	double baselineError = this.baseline.getError(dim_id);
		248	double currentError = this.current.getError(dim_id);
		249	this.error = Double.isNaN(baselineError)
		250	? currentError / baselineValue
		251	: Math.sqrt(baselineErrorbaselineError + currentErrorcurrentError) / baselineValue;
		252	}
		253	}
		254
		255	// Set the failure on the current build if necessary
		256	int failure_threshold = getPerformance().failure_threshold;
		257	if (this.delta >= (failure_threshold/100.0)) {
		258	StringBuffer buffer = new StringBuffer("Performance criteria not met when compared to '"); //$NON-NLS-1$
		259	buffer.append(this.baseline.getName());
		260	buffer.append("': "); //$NON-NLS-1$
		261	buffer.append(DEFAULT_DIM.getName());
		262	buffer.append("= "); //$NON-NLS-1$
		263	buffer.append(timeString((long)this.current.getValue()));
		264	buffer.append(" is not within [0%, "); //$NON-NLS-1$
		265	buffer.append(100+failure_threshold);
		266	buffer.append("'%] of "); //$NON-NLS-1$
		267	buffer.append(timeString((long)this.baseline.getValue()));
		268	this.current.setFailure(buffer.toString());
		269	}
247	}	270	}
248		271
249	/**	272	/**

Lines 51-68 Link Here

(-)src/org/eclipse/test/internal/performance/results/AbstractResults.java (-3 / +4 lines)
51	InternalDimensions.CPU_TIME	51	InternalDimensions.CPU_TIME
52	};	52	};
53		53
		54	static final int DEFAULT_DIM_INDEX = 0;
54	/**	55	/**
55	* The default dimension used to display results (typically in fingerprints).	56	* The default dimension used to display results (typically in fingerprints).
56	* <p>	57	* <p>
57	* Currently {@link InternalDimensions#ELAPSED_PROCESS}	58	* Currently {@link InternalDimensions#ELAPSED_PROCESS}
58	*/	59	*/
59	public static final Dim DEFAULT_DIM = SUPPORTED_DIMS[0];	60	public static final Dim DEFAULT_DIM = SUPPORTED_DIMS[DEFAULT_DIM_INDEX];
60		61
61	/**	62	/**
62	* The list of possible configurations.	63	* The list of possible configurations.
63	* <p>	64	* <p>
64	* Only used if no specific configurations are specified	65	* Only used if no specific configurations are specified
65	* (see {@link PerformanceResults#read(String[][], String)}.	66	* (see {@link PerformanceResults#read(File)}.
66	*/	67	*/
67	public final static String[] CONFIGS;	68	public final static String[] CONFIGS;
68		69
Lines 70-76 Link Here
70	* The list of possible test boxes.	71	* The list of possible test boxes.
71	* <p>	72	* <p>
72	* Only used if no specific configurations are specified	73	* Only used if no specific configurations are specified
73	* (see {@link PerformanceResults#read(String[][], String)}.	74	* (see {@link PerformanceResults#read(File)}.
74	*/	75	*/
75	public final static String[] BOXES = {	76	public final static String[] BOXES = {
76	"RHEL 4.0 Sun 1.4.2_10 (3 GHz 2.5 GB)", //$NON-NLS-1$	77	"RHEL 4.0 Sun 1.4.2_10 (3 GHz 2.5 GB)", //$NON-NLS-1$

Lines 27-43 Link Here

(-)src/org/eclipse/test/internal/performance/results/BuildResults.java (-7 / +18 lines)
27		27
28	// Build information	28	// Build information
29	String date;	29	String date;
30	boolean baseline;
31	String failure;
32	String comment;	30	String comment;
33	int summaryKind = -1;	31	int summaryKind = -1;
34		32
35	// Dimensions informations	33	// Dimensions information
36	Dim[] dimensions;	34	Dim[] dimensions;
37	double[] average, stddev;	35	double[] average, stddev;
38	long[] count;	36	long[] count;
39	double[][] values;	37	double[][] values;
40	BuildResults(AbstractResults parent) {	38
		39	// Comparison information
		40	boolean baseline;
		41	String failure;
		42
		43	BuildResults(AbstractResults parent) {
41	super(parent, -1);	44	super(parent, -1);
42	}	45	}
43		46
Lines 195-203 Link Here
195	* @return The failure message or <code>null</null> if the scenario passed.	198	* @return The failure message or <code>null</null> if the scenario passed.
196	*/	199	*/
197	public String getFailure() {	200	public String getFailure() {
198	if (this.failure == null) {	201	if (this.comment != null) return this.comment;
199	return null;
200	}
201	return this.failure;	202	return this.failure;
202	}	203	}
203		204
Lines 213-218 Link Here
213	}	214	}
214		215
215	/**	216	/**
		217	* Return the value of the performance result stored
		218	* for the default dimension of the current build.
		219	*
		220	* @return The value of the performance result
		221	*/
		222	public double getValue() {
		223	return this.average[DEFAULT_DIM_INDEX];
		224	}
		225
		226	/**
216	* Returns whether the build is a baseline build or not.	227	* Returns whether the build is a baseline build or not.
217	*	228	*
218	* @return <code>true</code> if the build name starts with the baseline prefix	229	* @return <code>true</code> if the build name starts with the baseline prefix