Attachment #69335 for bug #185580




import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Map;
import java.util.StringTokenizer;

import junit.framework.AssertionFailedError;

import org.eclipse.test.internal.performance.data.Dim;
import org.eclipse.test.internal.performance.db.DB;
import org.eclipse.test.internal.performance.db.Scenario;
import org.eclipse.test.internal.performance.db.SummaryEntry;
import org.eclipse.test.internal.performance.db.TimeSeries;
import org.eclipse.test.internal.performance.db.Variations;
import org.eclipse.test.internal.performance.eval.StatisticsUtil;
import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
import org.eclipse.test.performance.Dimension;




	public final static double STANDARD_ERROR_THRESHOLD = 0.03; // 3%
	static final NumberFormat PERCENT_FORMAT = NumberFormat.getPercentInstance();
	static final DecimalFormat DEVIATION_FORMAT = (DecimalFormat) NumberFormat.getPercentInstance();
	static {
//		DEVIATION_FORMAT.setMinimumIntegerDigits(2);
		DEVIATION_FORMAT.setMaximumFractionDigits(1);
		DEVIATION_FORMAT.setMinimumFractionDigits(1);
		DEVIATION_FORMAT.setPositivePrefix("+");
		DEVIATION_FORMAT.setNegativePrefix("- ");
	}
	static final DecimalFormat STDERR_FORMAT = (DecimalFormat) NumberFormat.getNumberInstance();
	static {
		STDERR_FORMAT.setMaximumFractionDigits(1);
		STDERR_FORMAT.setMinimumFractionDigits(1);
		STDERR_FORMAT.setMultiplier(100);
	}
	public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);
	public final static String STANDARD_ERROR_MESSAGE="Standard error on this test is higher than "+STANDARD_ERROR_THRESHOLD_STRING;
	public final static String OK_IMAGE="OK.gif";
	public final static String OK_IMAGE_WARN="OK_caution.gif";
	public final static String FAIL_IMAGE="FAIL.gif";
	public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";
	public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";
	public final static int OK = 0;
	public final static int WARN = 0x1;
	public final static int ERR = 0x2;
//	public final static int TTEST = 0x2;
	public final static int DEV = 0x4;

	/**


		while (tokenizer.hasMoreTokens()) {
			String labelDescriptor = tokenizer.nextToken();
			String[] elements = labelDescriptor.trim().split(",");
			ConfigDescriptor descriptor = new ConfigDescriptor(elements[0], elements[1]);
			configMap.put(elements[0], descriptor);
		}

		return componentNames;
	}

	/*
	 * @param fp -
	 *            a FingerPrint object
	 * @return - an html representation of the fingerprint.
	 *
	public static String getImageMap(FingerPrint fp) {
		String componentDescription = fp.configDescriptor.description;
		String areas = fp.bar.getAreas();

		}
		return output;
	}
	*/

	/**
	 * Utility method to copy a file.

			out.close();
		}
	}
	
	public static double[] resultStats(Variations variations, String scenarioName, String baseline, String config) {
		String current = (String) variations.get(PerformanceTestPlugin.BUILD);
		Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;
		Scenario newScenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, baseline, current, new Dim[] { significanceDimension });
        String[] timeSeriesLabels= newScenario.getTimeSeriesLabels();
        TimeSeries timeSeries = newScenario.getTimeSeries(significanceDimension);
        boolean hasBaseline = timeSeriesLabels.length == 2 && timeSeriesLabels[0].equals(baseline);
        long buildTime = hasBaseline ? 0 : getBuildTime(timeSeriesLabels[0]);
        double[] results = resultsStatistics(timeSeries, buildTime);
        return results;
	}

	public static long getBuildTime(String buildName) {
	    long time = 0;
        try {
			if (buildName.startsWith("3.2")) {
				int length = buildName.length();
				SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddhhmm");
				Date date = dateFormat.parse(buildName.substring(length-12, length));
				time = date.getTime();
			} else {
				SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd-hhmm");
				Date date = dateFormat.parse(buildName.substring(1));
				time = -date.getTime();
			}
        } catch(ParseException pe) {
        	// skip
        }
        return time;
    }

    private static double[] resultsStatistics(TimeSeries timeSeries, long buildTime) {
    	try {
	    	double[] values = new double[] { timeSeries.getValue(0), timeSeries.getValue(1) };
	    	long[] counts = new long[] { timeSeries.getCount(0), timeSeries.getCount(1) };
	    	double[] stddevs = new double[] { timeSeries.getStddev(0), timeSeries.getStddev(1) };
	    	double stderr = standardError(values, stddevs, counts);
			return new double[] {
				StatisticsUtil.deviation(values),
				stderr,
				buildTime,
			};
    	}
    	catch (ArrayIndexOutOfBoundsException aioobe) {
    		return null;
    	}
	}

    private static double standardError(double[] values, double[] stddevs, long[] counts) {
    	if (counts[0] == 1 || counts[1] == 1) {
    		return Double.NaN;
    	}
    	return Math.sqrt((stddevs[0] * stddevs[0] / counts[0]) + (stddevs[1] * stddevs[1] / counts[1])) / values[0];
    }

	public static boolean hasConfidentResult(TimeSeries timeSeries, long buildTime) {
	    double[] resultStats = resultsStatistics(timeSeries, buildTime);
	    return (confidenceLevel(resultStats) & ERR) == 0;
    }
	public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) {

	public static String failureMessage(double[] resultStats, boolean full) {
		StringBuffer buffer = new StringBuffer();
		int level = confidenceLevel(resultStats);
//		boolean isWarn = (level & WARN) != 0;
		boolean isErr = (level & ERR) != 0;
		if (full) {
			if (isErr) {
				buffer.append("*** WARNING ***  ");
	 			buffer.append(Messages.bind(Messages.standardError, PERCENT_FORMAT.format(resultStats[1]), STANDARD_ERROR_THRESHOLD_STRING));
			}
			return buffer.toString();
		}
		if (!full) buffer.append("<font color=\"#0000FF\" size=\"1\">  ");
		if (resultStats != null) {
			double deviation = resultStats[0];
			if (Double.isNaN(deviation) || Double.isInfinite(deviation)) {
				buffer.append("<font color=\"red\" size=\"1\"> ");
	 			buffer.append("[n/a]");
 			} else {
 				double stderr = resultStats[1];
				deviation = Math.abs(deviation)<0.001 ? 0 : -deviation;
				buffer.append("<font color=\"#0000FF\" size=\"1\">");
	 			if (Double.isNaN(stderr) || Double.isInfinite(stderr)) {
		 			buffer.append(DEVIATION_FORMAT.format(deviation));
					buffer.append("</font><font color=\"#DDDD00\" size=\"1\"> ");
		 			buffer.append(" [n/a]");
	 			} else {
		 			buffer.append(DEVIATION_FORMAT.format(deviation));
	 				buffer.append(" [&#177;");
	 				buffer.append(STDERR_FORMAT.format(Math.abs(stderr)));
	 				buffer.append(']');
	 			}
 			}
			buffer.append("</font>");
		}
		if (!full) buffer.append("</font>");
		return buffer.toString();
	}
	public static int confidenceLevel(double[] resultStats) {
		int level = OK;
 		if (resultStats != null){
 			if (resultStats[1] >= (Utils.STANDARD_ERROR_THRESHOLD/2)) { // warns standard error higher than the half of authorized threshold
 				level |= WARN;
// 			}
 			if (resultStats[2] > 0) { // signal standard error higher than 0% (only one iteration)
 				level |= SIGN;
 			}
 			if (resultStats[1] >= Utils.STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold
 				level |= ERR;
 			}
 		}

		return name.equals(pattern);
	}

	public static String getImage(int confidence, boolean hasExplanation) {
	    boolean scenarioFailed = (confidence & DEV) != 0;
	    String image = null;


	    }
	    return image;
    }

	public static boolean hasSummary(SummaryEntry[] summaries, String scenarioName) {
		int length = summaries == null ? 0 : summaries.length;
		for (int i=0; i<length; i++) {
			SummaryEntry summary = summaries[i];
			if (summary.scenarioName.equals(scenarioName)) {
				return true;
			}
		}
		return false;
	}

	public static String getScenarioShortName(String scenarioName, int max) {

		// Remove class name qualification
		int testSeparator = scenarioName.indexOf('#');
		if (testSeparator <= 0) {
			testSeparator = scenarioName.lastIndexOf('.');
			if (testSeparator <= 0) {
				if (max > 0 && scenarioName.length() > max) {
					return "*"+scenarioName.substring(0, max);
				}
				return scenarioName;
			}
		}
		int classSeparator = scenarioName.substring(0, testSeparator).lastIndexOf('.');
		if (classSeparator < 0) {
			if (max > 0 && scenarioName.length() > max) {
				return "*"+scenarioName.substring(0, max);
			}
			return scenarioName;
		}
		int length = scenarioName.length();
		String shortName = scenarioName.substring(classSeparator+1, length);

		// Remove qualification from test name
		StringTokenizer tokenizer = new StringTokenizer(shortName, " :,", true);
		StringBuffer buffer = new StringBuffer(tokenizer.nextToken());
		while (tokenizer.hasMoreTokens()) {
			String token = tokenizer.nextToken();
			char fc = token.charAt(0);
			while (fc == ' ' || fc == ',' || fc == ':') {
				buffer.append(token); // add the separator
				token = tokenizer.nextToken();
				fc = token.charAt(0);
			}
			int last = token .lastIndexOf('.');
			if (last >= 3) {
				int first = token .indexOf('.');
				if (first == last) {
					buffer.append(token);
				} else {
//					buffer.append(token.substring(0, first));
//					buffer.append("...");
					buffer.append(token.substring(last+1));
				}
			} else {
				buffer.append(token);
			}
		}
		if (max > 0 && buffer.length() > max) {
			return "*"+buffer.substring(0, max);
		}
		return buffer.toString();
	}
}




 *******************************************************************************/
package org.eclipse.test.performance.ui;

import java.io.PrintStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Hashtable;
import java.util.StringTokenizer;

import org.eclipse.test.internal.performance.PerformanceTestPlugin;
import org.eclipse.test.internal.performance.db.DB;
import org.eclipse.test.internal.performance.db.Scenario;
import org.eclipse.test.internal.performance.db.SummaryEntry;
import org.eclipse.test.internal.performance.db.Variations;

public class ScenarioStatusTable {

	private String scenarioPattern;
	private ArrayList configNames=new ArrayList();
	private Hashtable scenarioComments;
	private SummaryEntry[] fingerprintEntries;
	private final String baseline;
	
	private class ScenarioStatus{
		Hashtable statusMap;
		String name, shortName;
		Hashtable configStatus;
		Hashtable resultsMap;
		boolean hasSlowDownExplanation=false;
		boolean fingerprint = false;
		boolean hasBaseline = true;
						
		public ScenarioStatus(String scenarioName){
			name = scenarioName;
			statusMap = new Hashtable();
			configStatus = new Hashtable();
			resultsMap = new Hashtable();
		}

	}

	 * @param scenarioPattern
	 * @param configDescriptors
	 */
	public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, SummaryEntry[] fpSummaries, String baseline){
		configMaps=configDescriptors;
		this.variations=variations;
		this.scenarioPattern=scenarioPattern;
		this.scenarioComments=scenarioComments;
		this.baseline= baseline;
		this.fingerprintEntries = fpSummaries;
	}
	
		/**
    	 * Prints the HTML representation of scenario status table into the given stream.
    	 */
    	public void print(PrintStream stream, boolean filter) {
    		String OS="config";
            Scenario[] scenarios= DB.queryScenarios(variations, scenarioPattern, OS, null);
    	
    		if (scenarios != null && scenarios.length > 0) {
    			ArrayList scenarioStatusList=new ArrayList();
    
    			for (int i= 0; i < scenarios.length; i++) {
    				Scenario scenario= scenarios[i];
    				String scenarioName=scenario.getScenarioName();
    				if (filter && !Utils.matchPattern(scenarioName, scenarioPattern)) continue;
    				// returns the config names. Making assumption that indices in
    				// the configs array map to the indices of the failure messages.
    				String[] configs=scenario.getTimeSeriesLabels();
    				String[] failureMessages= scenario.getFailureMessages();
    				ScenarioStatus scenarioStatus=new ScenarioStatus(scenarioName);
    				scenarioStatus.fingerprint = Utils.hasSummary(this.fingerprintEntries, scenarioName);
				scenarioStatusList.add(scenarioStatus);

				String scenarioComment= (String)scenarioComments.get(scenarioName);
				if (scenarioComment != null)
					scenarioStatus.hasSlowDownExplanation= true;
				
				for (int j=0;j<configs.length;j++){
					if (!configNames.contains(configs[j]))
						configNames.add(configs[j]);

					double[] resultStats = Utils.resultStats(variations, scenarioName, baseline, configs[j]);
					int confidenceLevel = Utils.confidenceLevel(resultStats);
					
					boolean hasScenarioFailure = failureMessages[j] != null && failureMessages[j].indexOf(configs[j]) != -1; // ensure correct failure message relates to config
					StringBuffer buffer = new StringBuffer();
					if (hasScenarioFailure) {
						buffer.append(failureMessages[j]);
						if (scenarioStatus.hasSlowDownExplanation) {
							buffer.append(" - Explanation comment: ");
							buffer.append(scenarioComment);
						}
						confidenceLevel |= Utils.DEV;
					}

    				String scenarioComment= (String)scenarioComments.get(scenarioName);
    				if (scenarioComment != null)
    					scenarioStatus.hasSlowDownExplanation= true;
    
    				int confsLength = configs.length;
    				for (int j=0; j<confsLength; j++){
    					if (!configNames.contains(configs[j]))
    						configNames.add(configs[j]);
    
    					Variations v = (Variations) variations.clone();
    					v.put(PerformanceTestPlugin.CONFIG, configs[j]);
    					double[] resultStats = Utils.resultStats(v, scenarioName, baseline, configs[j]);
    					if (resultStats == null) continue;
    					if (resultStats != null && resultStats[1] < 0 && scenarioStatus.hasBaseline) scenarioStatus.hasBaseline = false;
    					int confidenceLevel = Utils.confidenceLevel(resultStats);
    					
    					boolean hasScenarioFailure = failureMessages[j] != null && failureMessages[j].indexOf(configs[j]) != -1; // ensure correct failure message relates to config
    					StringBuffer buffer = new StringBuffer();
    					if (hasScenarioFailure) {
    						buffer.append(failureMessages[j]);
    						if (scenarioStatus.hasSlowDownExplanation) {
    							buffer.append(" - Explanation comment: ");
    							buffer.append(scenarioComment);
    						}
    						confidenceLevel |= Utils.DEV;
    					}
    
    					String text = Utils.failureMessage(resultStats, false);
    					if (text == null) continue;
    					scenarioStatus.configStatus.put(configs[j], new Integer(confidenceLevel));
    					scenarioStatus.statusMap.put(configs[j], buffer.toString());
    					scenarioStatus.resultsMap.put(configs[j], text);
    					if (scenarioStatus.shortName == null) {
    						long time = (long) resultStats[2];
    						if (time == 0) { // baseline is OK
								scenarioStatus.shortName = Utils.getScenarioShortName(scenarioName, -1);
    						} else {
    							StringBuffer shortName = new StringBuffer("*");
    							shortName.append(Utils.getScenarioShortName(scenarioName, -1));
    							shortName.append(" <small>(vs.&nbsp;");
    							if (time < 0) {
									SimpleDateFormat dateFormat = new SimpleDateFormat("3.2_200606291905_yyyyMMddhhmm");
    								shortName.append(dateFormat.format(new Date(-time)));
    							} else {
									SimpleDateFormat dateFormat = new SimpleDateFormat("IyyyyMMdd-hhmm");
    								shortName.append(dateFormat.format(new Date(time)));
    							}
    							shortName.append(")</small>");
								scenarioStatus.shortName = shortName.toString();
    						}
    					}
    					
    					if (!scenarioStatusList.contains(scenarioStatus)) {
    						scenarioStatusList.add(scenarioStatus);
    					}
    				}
    			}
    			
    			String label=null;
    			stream.println("<br><h4>Scenario Status</h4>");
    			stream.println("The scenario status table shows all scenarios tests result for all performance test machines.");
    			stream.println("It gives a complete but compact view of performance result for the component.<br>");
    			stream.println("For each test (ie. in each cell of this table), following information are displayed:");
    			stream.println("<ul>");
    			stream.println("<li>an icon showing whether the test fails or passes and whether it's reliable or not.<br>");
    			stream.println("The legend for this icon is:");
    			stream.println("<ul>");
    			stream.print("<li>Green (<img src=\"");
    			stream.print(Utils.OK_IMAGE);
    			stream.print("\">): mark a <b>successful result</b>, which means this test has neither significant performance regression nor significant standard error</li>");
    			stream.print("<li>Red (<img src=\"");
    			stream.print(Utils.FAIL_IMAGE);
    			stream.println("\">): mark a <b>failing result</b>, which means this test shows a significant performance regression (more than 10%)</li>");
    			stream.print("<li>Gray (<img src=\"");
    			stream.print(Utils.FAIL_IMAGE_EXPLAINED);
    			stream.println("\">): mark a <b>failing result</b> (see above) with a comment explaining this degradation.</li>");
    			stream.print("<li>Yellow (<img src=\"");
    			stream.print(Utils.FAIL_IMAGE_WARN);
    			stream.print("\"> or <img src=\"");
    			stream.print(Utils.OK_IMAGE_WARN);
    			stream.print("\">): mark a <b>failing or successful result</b> with a significant standard error (more than ");
    			stream.print(Utils.STANDARD_ERROR_THRESHOLD_STRING);
    			stream.println(")</li>");
    			stream.println("<li>\"n/a\": mark a test for with <b>no</b> performance results</li>");
    			stream.println("</ul></li>");
    			stream.println("<li>the value of the deviation from the baseline as a percentage (ie. formula is: <code>(build_test_time - baseline_test_time) / baseline_test_time</code>)</li>");
    			stream.println("<li>the value of the standard error of this deviation as a percentage (ie. formula is: <code>sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time</code>)<br>");
    			stream.println("Tests with only one iteration have no available error (replaced by \"[n/a]\".</li>");
    			stream.println("</ul>");
    			stream.println("<u>Hints</u>:<ul>");
    			stream.println("<li>fly over image of failing tests to see the complete error message</li>");
    			stream.println("<li>to look at the complete and detailed test results, click on its image</li>");
    			stream.println("<li>scenario name may be <b>emphazised</b> when the results are also displayed in the fingerprint</li>");
    			stream.println("<li>scenario name may start with an '*' when the scenario has no results in the last baseline run</li>");
    			stream.println("<li>standard error may be '<font color=\"#CCCC00\">[n/a]</font>' when it cannot be computed, typically when the test has only one measure!</li>");
    			stream.println("</ul>");
    			stream.println();
    			stream.println("<table border=\"1\">");
    			stream.println("<tr>");
    			stream.print("<td><h4>All ");
    			stream.print(scenarios.length);
    			stream.println(" scenarios</h4></td>");
    
    			for (int i= 0; i < configNames.size(); i++){
    				label = configNames.get(i).toString();
    				String columnTitle = label;
    				if (configMaps!=null) {
    					Utils.ConfigDescriptor configDescriptor= (Utils.ConfigDescriptor)configMaps.get(label);
    					if (configDescriptor != null) {
    						int idx = configDescriptor.description.indexOf('(');
    						if (idx < 0) {
    							columnTitle = configDescriptor.description;
    						} else {
    							// first line
	    						StringTokenizer tokenizer = new StringTokenizer(configDescriptor.description.substring(0, idx).trim(), " ");
	    						StringBuffer buffer = new StringBuffer(tokenizer.nextToken());
	    						while (tokenizer.hasMoreTokens()) {
	    							buffer.append("&nbsp;");
	    							buffer.append(tokenizer.nextToken());
	    						}
	    						buffer.append(' ');
	    						// second line
	    						tokenizer = new StringTokenizer(configDescriptor.description.substring(idx).trim(), " ");
	    						buffer.append(tokenizer.nextToken());
	    						while (tokenizer.hasMoreTokens()) {
	    							buffer.append("&nbsp;");
	    							buffer.append(tokenizer.nextToken());
	    						}
	    						columnTitle = buffer.toString();
    						}
    					}
    				}
    				stream.print("<td><h5>");
    				stream.print(columnTitle);
    				stream.println("</h5>");
    			}
    			
    			// counter for js class Id's
    			int jsIdCount=0;
    			for (int j= 0; j < scenarioStatusList.size(); j++) {
    				
    				ScenarioStatus status=(ScenarioStatus)scenarioStatusList.get(j);
    
    				stream.println("<tr>");
    				stream.print("<td>");
    				if (status.fingerprint) stream.print("<b>");
    				if (!status.hasBaseline) stream.print("*");
    //				stream.print(status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1));
    				stream.print(status.shortName);
    				if (!status.hasBaseline) stream.print("</i>");
    				if (status.fingerprint) stream.print("</b>");
    				stream.println();
    				for (int i=0;i<configNames.size();i++){
    					String configName=configNames.get(i).toString();
    					String aUrl=configName;
    					if (status.statusMap.containsKey(configName)){
    						String message = (String) status.statusMap.get(configName);
    						int confidence = ((Integer) status.configStatus.get(configName)).intValue();
    						String image = Utils.getImage(confidence, status.hasSlowDownExplanation);
    						stream.print("<td><a ");
    						if ((confidence & Utils.DEV) == 0 || message.length() == 0){
    							// write deviation with error in table when test pass
    							stream.print("href=\"");
    							stream.print(aUrl);
    							stream.print('/');
    							stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
    							stream.println(".html\">");
    							stream.print("<img hspace=\"10\" border=\"0\" src=\"");
    							stream.print(image);
    							stream.println("\"/></a>");
    						} else {
    							// create message with tooltip text including deviation with error plus failure message
    							jsIdCount+=1;
    							stream.print("class=\"tooltipSource\" onMouseover=\"show_element('toolTip");
    							stream.print(jsIdCount);
    							stream.print("')\" onMouseout=\"hide_element('toolTip");
    							stream.print(jsIdCount);
    							stream.print("')\" \nhref=\"");
    							stream.print(aUrl);
    							stream.print('/');
    							stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
    							stream.println(".html\">");
    							stream.print("<img hspace=\"10\" border=\"0\" src=\"");
    							stream.print(image);
    							stream.println("\"/>");
    							stream.print("<span class=\"hidden_tooltip\" id=\"toolTip");
    							stream.print(jsIdCount);
    							stream.print("\">");
    							stream.print(message);
    							stream.println("</span></a>");
    						}
    						String results = (String) status.resultsMap.get(configName);
    						if (results != null) stream.println(results);
    					}else{
    						stream.println("<td>n/a");
    					}
    				}
    				stream.flush();
    			}
    			stream.append("</table>");
    		}
        }
}




import java.util.Hashtable;

import org.eclipse.swt.SWT;
import org.eclipse.swt.events.PaintEvent;
import org.eclipse.swt.events.PaintListener;
import org.eclipse.swt.graphics.GC;
import org.eclipse.swt.graphics.Image;
import org.eclipse.swt.graphics.ImageData;
import org.eclipse.swt.graphics.ImageLoader;
import org.eclipse.swt.graphics.Point;
import org.eclipse.swt.layout.FillLayout;
import org.eclipse.swt.widgets.Canvas;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.test.internal.performance.PerformanceTestPlugin;
import org.eclipse.test.internal.performance.data.Dim;
import org.eclipse.test.internal.performance.db.DB;

import org.eclipse.test.internal.performance.db.SummaryEntry;
import org.eclipse.test.internal.performance.db.TimeSeries;
import org.eclipse.test.internal.performance.db.Variations;
import org.eclipse.test.internal.performance.eval.StatisticsUtil;
import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
import org.eclipse.test.performance.Dimension;
import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;



        if (component==null){
        	entries= DB.querySummaries(variations,null);
        	this.component="";
        } else {
        else
        	entries=DB.querySummaries(variations,component+'%');
        }
       	run();
    }    
    
    /**
     * Creates the fingerprint gif, image map and scenario status table for the component.
     * @param entries - the result of a database query for summaries for a specified variation.
     */
    public void run() {
        new File(outputDirectory).mkdirs();
        String referenceName=referenceBuildId;
        String currentName=currentBuildId;

                
        if (entries != null) {
            for (int i= 0; i < entries.length; i++) {
                SummaryEntry summary = entries[i];
                if (summary.comment==null)
                	add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName);
                else{
                	setComment(summary.scenarioName, summary.comment);
                	add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName, summary.comment);
                }
            }
        }

               
        if (component=="")
        	outName= "FP_"+referenceName + '_' + currentBuildId+"."+configDescriptor.name;
        save(outputDirectory + '/' + outName);
        
        //show(bar);
     

    	scenarioComments.put(scenario,comment);
    }

    private void add(String name, Dim[] dims, String scenarioName) {
    	add (name,dims,scenarioName,null);
    }

    private void add(String name, Dim[] dims, String scenarioName, String comment) {
         String refData= "";
        Scenario scenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, referenceBuildId, currentBuildId, dims);
        String[] timeSeriesLabels= scenario.getTimeSeriesLabels();
        long buildTime = 0;
        if (timeSeriesLabels.length == 2) {
            // we mark the label with a '*' or '' to indicate that no data was available for the specified builds
            if (!timeSeriesLabels[0].equals(referenceBuildId)) {
                name= '*' + name;
                refData= " (" + timeSeriesLabels[0] + ")";
                buildTime = Utils.getBuildTime(timeSeriesLabels[0]);
            } else if (!timeSeriesLabels[1].equals(currentBuildId)) {
                name= '' + name;
                refData= " (" + timeSeriesLabels[1] + ")";

            	double percent= 0.0;
            	boolean hasConfidentResult= true;
            	if (l > 1) {
//            		hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name);
            		hasConfidentResult= Utils.hasConfidentResult(timeSeries, buildTime);
            		/*if (!rejectNullHypothesis) {
            			NumberFormat percentFormatter= NumberFormat.getPercentInstance();
            			String statisticsComment= "There is not enough evidence to reject the null hypothesis at the " + percentFormatter.format(percentile.inside()) + "level";

	         
    }

    private void save(String output) {

    	//if (bar.getFItems().size()==0)
    		//return;

    
    /*
     * Displays bar graph in window
     *
    private void show(final BarGraph bar) {
        Display display= new Display();
        

	public String getOutName() {
		return outName;
	}
    */

	/**
	 * @return - an html representation of the fingerprint.
	 */
	public String getImageMap() {
		String componentDescription = this.configDescriptor.description;
		String areas = this.bar.getAreas();
		if (areas == null)
			areas = "";
		String output = "";
		if (new File(this.outputDirectory, this.outName + ".gif").exists()) {
			output = "<h4>" + componentDescription + "</h4>";
			output = output.concat("<img src=\"" + this.outName + ".gif\" usemap=\"#" + this.outName + "\">" + "<map name=\"" + this.outName + "\">" + areas + "</map>\n");
		} else {
			output = output.concat("<br><br>There is no fingerprint for " + componentDescription + "<br><br>\n");
		}
		return output;
	}
}

Lines 25-30 Link Here

(-)src/org/eclipse/test/performance/ui/Main.java (-9 / +15 lines)
25	import org.eclipse.core.runtime.Platform;	25	import org.eclipse.core.runtime.Platform;
26		26
27	import org.eclipse.test.internal.performance.db.Scenario;	27	import org.eclipse.test.internal.performance.db.Scenario;
		28	import org.eclipse.test.internal.performance.db.SummaryEntry;
28	import org.eclipse.test.internal.performance.db.Variations;	29	import org.eclipse.test.internal.performance.db.Variations;
29	import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;	30	import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;
30		31
Lines 87-93 Link Here
87	os.println(Utils.HTML_DEFAULT_CSS);	88	os.println(Utils.HTML_DEFAULT_CSS);
88	os.println("<body>");	89	os.println("<body>");
89	Hashtable fps = (Hashtable) fingerPrints.get(component);	90	Hashtable fps = (Hashtable) fingerPrints.get(component);
90	Enumeration configs = fps.keys();
91		91
92	int baselineUnderScoreIndex=baseline.indexOf("_");	92	int baselineUnderScoreIndex=baseline.indexOf("_");
93	int currentUnderScoreIndex=currentBuildId.indexOf("_");	93	int currentUnderScoreIndex=currentBuildId.indexOf("_");
Lines 106-122 Link Here
106	os.println(title);	106	os.println(title);
107		107
108	//print the html representation of fingerprint for each config	108	//print the html representation of fingerprint for each config
		109	Enumeration configs = fps.keys();
		110	SummaryEntry[] fpSummaries = null;
109	while (configs.hasMoreElements()) {	111	while (configs.hasMoreElements()) {
110	String config = configs.nextElement().toString();	112	String config = configs.nextElement().toString();
111	FingerPrint fp = (FingerPrint) fps.get(config);	113	FingerPrint fingerPrint = (FingerPrint) fps.get(config);
112	os.println(Utils.getImageMap(fp));	114	os.println(fingerPrint.getImageMap());
		115	if (fpSummaries == null) {
		116	fpSummaries = fingerPrint.entries;
		117	}
113	}	118	}
114	if (component != "") {	119	if (component.length() > 0) {
115	//print the component scenario status table beneath the fingerprint	120	//print the component scenario status table beneath the fingerprint
116	variations.put("config", "%");	121	variations.put("config", "%");
117	ScenarioStatusTable sst = new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, baseline);	122	boolean filter = false; // use scenario filter to minimize DB requests while testing...
118	// ScenarioStatusTable sst = new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, baseline);	123	ScenarioStatusTable sst = filter
119	os.println(sst.toString());	124	? new ScenarioStatusTable(variations, this.scenarioFilter, configDescriptors,scenarioComments, fpSummaries, baseline)
		125	: new ScenarioStatusTable(variations, component + "%", configDescriptors,scenarioComments, fpSummaries, baseline);
		126	sst.print(os, filter);
120	}	127	}
121		128
122	os.println(Utils.HTML_CLOSE);	129	os.println(Utils.HTML_CLOSE);
Lines 292-299 Link Here
292	System.out.println("Missing value for -config.properties parameter");	299	System.out.println("Missing value for -config.properties parameter");
293	printUsage();	300	printUsage();
294	}	301	}
295	configDescriptors = Utils	302	configDescriptors = Utils.getConfigDescriptors(configProperties);
296	.getConfigDescriptors(configProperties);
297	i++;	303	i++;
298	continue;	304	continue;
299	}	305	}




/*******************************************************************************
 * Copyright (c) 2000, 2007 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBM Corporation - initial API and implementation
 *******************************************************************************/
package org.eclipse.test.performance.ui;

import java.text.MessageFormat;

import org.eclipse.osgi.util.NLS;

public class Messages extends NLS {

	private static final String BUNDLE_NAME = "org.eclipse.test.performance.ui.messages";//$NON-NLS-1$

	private Messages() {
		// Do not instantiate
	}

	public static String standardError;

	static {
		NLS.initializeMessages(BUNDLE_NAME, Messages.class);
	}
	
	/**
	 * Bind the given message's substitution locations with the given string values.
	 * 
	 * @param message the message to be manipulated
	 * @return the manipulated String
	 */
	public static String bind(String message) {
		return bind(message, null);
	}
	
	/**
	 * Bind the given message's substitution locations with the given string values.
	 * 
	 * @param message the message to be manipulated
	 * @param binding the object to be inserted into the message
	 * @return the manipulated String
	 */
	public static String bind(String message, Object binding) {
		return bind(message, new Object[] {binding});
	}

	/**
	 * Bind the given message's substitution locations with the given string values.
	 * 
	 * @param message the message to be manipulated
	 * @param binding1 An object to be inserted into the message
	 * @param binding2 A second object to be inserted into the message
	 * @return the manipulated String
	 */
	public static String bind(String message, Object binding1, Object binding2) {
		return bind(message, new Object[] {binding1, binding2});
	}

	/**
	 * Bind the given message's substitution locations with the given string values.
	 * 
	 * @param message the message to be manipulated
	 * @param bindings An array of objects to be inserted into the message
	 * @return the manipulated String
	 */
	public static String bind(String message, Object[] bindings) {
		return MessageFormat.format(message, bindings);
	}

}




###############################################################################
# Copyright (c) 2000, 2007 IBM Corporation and others.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
#
# Contributors:
#     IBM Corporation - initial API and implementation
###############################################################################

### Performance Tests Messages.

standardError = Standard error on this test is {0} (should be less than {1} to become reliable!)

Return to bug 185580

Lines 20-32 Link Here

(-)src/org/eclipse/test/performance/ui/Utils.java (-77 / +171 lines)
20	import java.io.InputStream;	20	import java.io.InputStream;
21	import java.io.OutputStream;	21	import java.io.OutputStream;
22	import java.io.PrintWriter;	22	import java.io.PrintWriter;
		23	import java.text.DecimalFormat;
23	import java.text.NumberFormat;	24	import java.text.NumberFormat;
		25	import java.text.ParseException;
		26	import java.text.SimpleDateFormat;
24	import java.util.ArrayList;	27	import java.util.ArrayList;
25	import java.util.Arrays;	28	import java.util.Arrays;
26	import java.util.Calendar;	29	import java.util.Calendar;
		30	import java.util.Date;
27	import java.util.HashMap;	31	import java.util.HashMap;
28	import java.util.Hashtable;	32	import java.util.Hashtable;
29	import java.util.Map;
30	import java.util.StringTokenizer;	33	import java.util.StringTokenizer;
31		34
32	import junit.framework.AssertionFailedError;	35	import junit.framework.AssertionFailedError;
Lines 43-52 Link Here
43	import org.eclipse.test.internal.performance.data.Dim;	46	import org.eclipse.test.internal.performance.data.Dim;
44	import org.eclipse.test.internal.performance.db.DB;	47	import org.eclipse.test.internal.performance.db.DB;
45	import org.eclipse.test.internal.performance.db.Scenario;	48	import org.eclipse.test.internal.performance.db.Scenario;
		49	import org.eclipse.test.internal.performance.db.SummaryEntry;
46	import org.eclipse.test.internal.performance.db.TimeSeries;	50	import org.eclipse.test.internal.performance.db.TimeSeries;
47	import org.eclipse.test.internal.performance.db.Variations;	51	import org.eclipse.test.internal.performance.db.Variations;
48	import org.eclipse.test.internal.performance.eval.StatisticsUtil;	52	import org.eclipse.test.internal.performance.eval.StatisticsUtil;
49	import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
50	import org.eclipse.test.performance.Dimension;	53	import org.eclipse.test.performance.Dimension;
51		54
52		55
Lines 54-78 Link Here
54		57
55	public final static double STANDARD_ERROR_THRESHOLD = 0.03; // 3%	58	public final static double STANDARD_ERROR_THRESHOLD = 0.03; // 3%
56	static final NumberFormat PERCENT_FORMAT = NumberFormat.getPercentInstance();	59	static final NumberFormat PERCENT_FORMAT = NumberFormat.getPercentInstance();
		60	static final DecimalFormat DEVIATION_FORMAT = (DecimalFormat) NumberFormat.getPercentInstance();
57	static {	61	static {
58	PERCENT_FORMAT.setMaximumFractionDigits(1);	62	// DEVIATION_FORMAT.setMinimumIntegerDigits(2);
		63	DEVIATION_FORMAT.setMaximumFractionDigits(1);
		64	DEVIATION_FORMAT.setMinimumFractionDigits(1);
		65	DEVIATION_FORMAT.setPositivePrefix("+");
		66	DEVIATION_FORMAT.setNegativePrefix("- ");
59	}	67	}
60	static final NumberFormat DOUBLE_FORMAT = NumberFormat.getNumberInstance();	68	static final DecimalFormat STDERR_FORMAT = (DecimalFormat) NumberFormat.getNumberInstance();
61	static {	69	static {
62	DOUBLE_FORMAT.setMaximumIntegerDigits(2);	70	STDERR_FORMAT.setMaximumFractionDigits(1);
63	DOUBLE_FORMAT.setMaximumFractionDigits(1);	71	STDERR_FORMAT.setMinimumFractionDigits(1);
		72	STDERR_FORMAT.setMultiplier(100);
64	}	73	}
65	public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);	74	public final static String STANDARD_ERROR_THRESHOLD_STRING = PERCENT_FORMAT.format(STANDARD_ERROR_THRESHOLD);
66	public final static String STANDARD_ERROR_MESSAGE="Standard error on this test is higher than "+STANDARD_ERROR_THRESHOLD_STRING;
67	public final static String OK_IMAGE="OK.gif";	75	public final static String OK_IMAGE="OK.gif";
68	public final static String OK_IMAGE_WARN="OK_caution.gif";	76	public final static String OK_IMAGE_WARN="OK_caution.gif";
69	public final static String FAIL_IMAGE="FAIL.gif";	77	public final static String FAIL_IMAGE="FAIL.gif";
70	public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";	78	public final static String FAIL_IMAGE_WARN="FAIL_caution.gif";
71	public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";	79	public final static String FAIL_IMAGE_EXPLAINED="FAIL_greyed.gif";
72	public final static int OK = 0;	80	public final static int OK = 0;
73	public final static int SIGN = 0x1;	81	public final static int WARN = 0x1;
74	public final static int ERR = 0x2;	82	public final static int ERR = 0x2;
75	// public final static int TTEST = 0x2;
76	public final static int DEV = 0x4;	83	public final static int DEV = 0x4;
77		84
78	/**	85	/**
Lines 207-213 Link Here
207		214
208	while (tokenizer.hasMoreTokens()) {	215	while (tokenizer.hasMoreTokens()) {
209	String labelDescriptor = tokenizer.nextToken();	216	String labelDescriptor = tokenizer.nextToken();
210	String[] elements = labelDescriptor.split(",");	217	String[] elements = labelDescriptor.trim().split(",");
211	ConfigDescriptor descriptor = new ConfigDescriptor(elements[0], elements[1]);	218	ConfigDescriptor descriptor = new ConfigDescriptor(elements[0], elements[1]);
212	configMap.put(elements[0], descriptor);	219	configMap.put(elements[0], descriptor);
213	}	220	}
Lines 268-278 Link Here
268	return componentNames;	275	return componentNames;
269	}	276	}
270		277
271	/**	278	/*
272	* @param fp -	279	* @param fp -
273	* a FingerPrint object	280	* a FingerPrint object
274	* @return - an html representation of the fingerprint.	281	* @return - an html representation of the fingerprint.
275	*/	282	*
276	public static String getImageMap(FingerPrint fp) {	283	public static String getImageMap(FingerPrint fp) {
277	String componentDescription = fp.configDescriptor.description;	284	String componentDescription = fp.configDescriptor.description;
278	String areas = fp.bar.getAreas();	285	String areas = fp.bar.getAreas();
Lines 287-292 Link Here
287	}	294	}
288	return output;	295	return output;
289	}	296	}
		297	*/
290		298
291	/**	299	/**
292	* Utility method to copy a file.	300	* Utility method to copy a file.
Lines 740-789 Link Here
740	out.close();	748	out.close();
741	}	749	}
742	}	750	}
743
744	public static double[] resultStats(Variations variations, String scenarioName, String baseline, String config) {	751	public static double[] resultStats(Variations variations, String scenarioName, String baseline, String config) {
745	String OS = "config";	752	String current = (String) variations.get(PerformanceTestPlugin.BUILD);
746		753	Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;
747	Variations tmpVariations=(Variations)variations.clone();	754	Scenario newScenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, baseline, current, new Dim[] { significanceDimension });
748	tmpVariations.put(OS,config);	755	String[] timeSeriesLabels= newScenario.getTimeSeriesLabels();
749	Scenario[] currentScenarios = DB.queryScenarios(tmpVariations, scenarioName,OS, null);	756	TimeSeries timeSeries = newScenario.getTimeSeries(significanceDimension);
750	Variations referenceVariations = (Variations) variations.clone();	757	boolean hasBaseline = timeSeriesLabels.length == 2 && timeSeriesLabels[0].equals(baseline);
751	referenceVariations.put(PerformanceTestPlugin.BUILD, baseline);	758	long buildTime = hasBaseline ? 0 : getBuildTime(timeSeriesLabels[0]);
752	referenceVariations.put(OS, config);	759	double[] results = resultsStatistics(timeSeries, buildTime);
753	Scenario[] refScenarios = DB.queryScenarios(referenceVariations,	760	return results;
754	scenarioName, OS, null);	761	}
755		762
756	Map referenceScenariosMap = new HashMap();	763	public static long getBuildTime(String buildName) {
757	Map currentScenariosMap = new HashMap();	764	long time = 0;
758	for (int i = 0; i < refScenarios.length; i++) {	765	try {
759	Scenario scenario = refScenarios[i];	766	if (buildName.startsWith("3.2")) {
760	String name = scenario.getScenarioName();	767	int length = buildName.length();
761	referenceScenariosMap.put(name, scenario);	768	SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddhhmm");
762	}	769	Date date = dateFormat.parse(buildName.substring(length-12, length));
763		770	time = date.getTime();
764	for (int i = 0; i < currentScenarios.length; i++) {	771	} else {
765	Scenario scenario = currentScenarios[i];	772	SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd-hhmm");
766	String name = scenario.getScenarioName();	773	Date date = dateFormat.parse(buildName.substring(1));
767	currentScenariosMap.put(name, scenario);	774	time = -date.getTime();
768	}	775	}
769	Percentile percentile = StatisticsUtil.T90;	776	} catch(ParseException pe) {
770	Scenario scenario = (Scenario) currentScenariosMap.get(scenarioName);	777	// skip
771		778	}
772	Scenario reference = (Scenario) referenceScenariosMap.get(scenarioName);	779	return time;
773	if (reference != null) {	780	}
774	// XXX have to find out the relevant dimension	781
775	Dim significanceDimension = (Dim) Dimension.ELAPSED_PROCESS;	782	private static double[] resultsStatistics(TimeSeries timeSeries, long buildTime) {
776	TimeSeries currentSeries = scenario.getTimeSeries(significanceDimension);	783	try {
777	TimeSeries baselineSeries = reference.getTimeSeries(significanceDimension);	784	double[] values = new double[] { timeSeries.getValue(0), timeSeries.getValue(1) };
778	if (currentSeries.getLength() > 0 && baselineSeries.getLength() > 0) {	785	long[] counts = new long[] { timeSeries.getCount(0), timeSeries.getCount(1) };
779	return StatisticsUtil.statisticsForTimeSeries(baselineSeries, 0, currentSeries, 0, percentile);	786	double[] stddevs = new double[] { timeSeries.getStddev(0), timeSeries.getStddev(1) };
780	}	787	double stderr = standardError(values, stddevs, counts);
781	}	788	return new double[] {
782	return null;	789	StatisticsUtil.deviation(values),
783	}	790	stderr,
		791	buildTime,
		792	};
		793	}
		794	catch (ArrayIndexOutOfBoundsException aioobe) {
		795	return null;
		796	}
		797	}
		798
		799	private static double standardError(double[] values, double[] stddevs, long[] counts) {
		800	if (counts[0] == 1 \|\| counts[1] == 1) {
		801	return Double.NaN;
		802	}
		803	return Math.sqrt((stddevs[0] * stddevs[0] / counts[0]) + (stddevs[1] * stddevs[1] / counts[1])) / values[0];
		804	}
784		805
785	public static boolean hasConfidentResult(Variations variations, String scenarioName, String baseline, String config) {	806	public static boolean hasConfidentResult(TimeSeries timeSeries, long buildTime) {
786	double[] resultStats = resultStats(variations, scenarioName, baseline, config);	807	double[] resultStats = resultsStatistics(timeSeries, buildTime);
787	return (confidenceLevel(resultStats) & ERR) == 0;	808	return (confidenceLevel(resultStats) & ERR) == 0;
788	}	809	}
789	public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) {	810	public static String failureMessage(Variations variations, String scenarioName, String baseline, String config) {
Lines 792-829 Link Here
792	public static String failureMessage(double[] resultStats, boolean full) {	813	public static String failureMessage(double[] resultStats, boolean full) {
793	StringBuffer buffer = new StringBuffer();	814	StringBuffer buffer = new StringBuffer();
794	int level = confidenceLevel(resultStats);	815	int level = confidenceLevel(resultStats);
795	boolean signal = (level & SIGN) != 0;	816	// boolean isWarn = (level & WARN) != 0;
796	boolean isErr = (level & ERR) != 0;	817	boolean isErr = (level & ERR) != 0;
797	if (full & isErr) {	818	if (full) {
798	buffer.append("* WARNING * ");	819	if (isErr) {
799	buffer.append(STANDARD_ERROR_MESSAGE);	820	buffer.append("* WARNING * ");
		821	buffer.append(Messages.bind(Messages.standardError, PERCENT_FORMAT.format(resultStats[1]), STANDARD_ERROR_THRESHOLD_STRING));
		822	}
		823	return buffer.toString();
800	}	824	}
801	if (!full) buffer.append("<font color=\"#0000FF\" size=\"1\"> ");
802	if (resultStats != null) {	825	if (resultStats != null) {
803	double deviation = resultStats[3]==0 ? 0 : -resultStats[3];	826	double deviation = resultStats[0];
804	if (deviation > 0) {	827	if (Double.isNaN(deviation) \|\| Double.isInfinite(deviation)) {
805	buffer.append('+');	828	buffer.append("<font color=\"red\" size=\"1\"> ");
806	}	829	buffer.append("[n/a]");
807	buffer.append(PERCENT_FORMAT.format(deviation));	830	} else {
808	if (signal) {	831	double stderr = resultStats[1];
809	buffer.append(" [±");	832	deviation = Math.abs(deviation)<0.001 ? 0 : -deviation;
810	buffer.append(DOUBLE_FORMAT.format(resultStats[2]*100));	833	buffer.append("<font color=\"#0000FF\" size=\"1\">");
811	buffer.append(']');	834	if (Double.isNaN(stderr) \|\| Double.isInfinite(stderr)) {
		835	buffer.append(DEVIATION_FORMAT.format(deviation));
		836	buffer.append("</font><font color=\"#DDDD00\" size=\"1\"> ");
		837	buffer.append(" [n/a]");
		838	} else {
		839	buffer.append(DEVIATION_FORMAT.format(deviation));
		840	buffer.append(" [±");
		841	buffer.append(STDERR_FORMAT.format(Math.abs(stderr)));
		842	buffer.append(']');
		843	}
812	}	844	}
		845	buffer.append("</font>");
813	}	846	}
814	if (!full) buffer.append("</font>");
815	return buffer.toString();	847	return buffer.toString();
816	}	848	}
817	public static int confidenceLevel(double[] resultStats) {	849	public static int confidenceLevel(double[] resultStats) {
818	int level = OK;	850	int level = OK;
819	if (resultStats != null){	851	if (resultStats != null){
820	// if (resultStats[1] >= 0 && resultStats[0] >= resultStats[1]) { // invalid t-test	852	if (resultStats[1] >= (Utils.STANDARD_ERROR_THRESHOLD/2)) { // warns standard error higher than the half of authorized threshold
821	// level \|= TTEST;	853	level \|= WARN;
822	// }
823	if (resultStats[2] > 0) { // signal standard error higher than 0% (only one iteration)
824	level \|= SIGN;
825	}	854	}
826	if (resultStats[2] >= Utils.STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold	855	if (resultStats[1] >= Utils.STANDARD_ERROR_THRESHOLD) { // standard error higher than the authorized threshold
827	level \|= ERR;	856	level \|= ERR;
828	}	857	}
829	}	858	}
Lines 861-867 Link Here
861	return name.equals(pattern);	890	return name.equals(pattern);
862	}	891	}
863		892
864	public static String getImage(int confidence, double[] resultStats, boolean hasExplanation) {	893	public static String getImage(int confidence, boolean hasExplanation) {
865	boolean scenarioFailed = (confidence & DEV) != 0;	894	boolean scenarioFailed = (confidence & DEV) != 0;
866	String image = null;	895	String image = null;
867		896
Lines 880-883 Link Here
880	}	909	}
881	return image;	910	return image;
882	}	911	}
883	}	912
		913	public static boolean hasSummary(SummaryEntry[] summaries, String scenarioName) {
		914	int length = summaries == null ? 0 : summaries.length;
		915	for (int i=0; i<length; i++) {
		916	SummaryEntry summary = summaries[i];
		917	if (summary.scenarioName.equals(scenarioName)) {
		918	return true;
		919	}
		920	}
		921	return false;
		922	}
		923
		924	public static String getScenarioShortName(String scenarioName, int max) {
		925
		926	// Remove class name qualification
		927	int testSeparator = scenarioName.indexOf('#');
		928	if (testSeparator <= 0) {
		929	testSeparator = scenarioName.lastIndexOf('.');
		930	if (testSeparator <= 0) {
		931	if (max > 0 && scenarioName.length() > max) {
		932	return "*"+scenarioName.substring(0, max);
		933	}
		934	return scenarioName;
		935	}
		936	}
		937	int classSeparator = scenarioName.substring(0, testSeparator).lastIndexOf('.');
		938	if (classSeparator < 0) {
		939	if (max > 0 && scenarioName.length() > max) {
		940	return "*"+scenarioName.substring(0, max);
		941	}
		942	return scenarioName;
		943	}
		944	int length = scenarioName.length();
		945	String shortName = scenarioName.substring(classSeparator+1, length);
		946
		947	// Remove qualification from test name
		948	StringTokenizer tokenizer = new StringTokenizer(shortName, " :,", true);
		949	StringBuffer buffer = new StringBuffer(tokenizer.nextToken());
		950	while (tokenizer.hasMoreTokens()) {
		951	String token = tokenizer.nextToken();
		952	char fc = token.charAt(0);
		953	while (fc == ' ' \|\| fc == ',' \|\| fc == ':') {
		954	buffer.append(token); // add the separator
		955	token = tokenizer.nextToken();
		956	fc = token.charAt(0);
		957	}
		958	int last = token .lastIndexOf('.');
		959	if (last >= 3) {
		960	int first = token .indexOf('.');
		961	if (first == last) {
		962	buffer.append(token);
		963	} else {
		964	// buffer.append(token.substring(0, first));
		965	// buffer.append("...");
		966	buffer.append(token.substring(last+1));
		967	}
		968	} else {
		969	buffer.append(token);
		970	}
		971	}
		972	if (max > 0 && buffer.length() > max) {
		973	return "*"+buffer.substring(0, max);
		974	}
		975	return buffer.toString();
		976	}
977	}

Lines 10-20 Link Here

(-)src/org/eclipse/test/performance/ui/ScenarioStatusTable.java (-152 / +239 lines)
10	*******************************************************************************/	10	*******************************************************************************/
11	package org.eclipse.test.performance.ui;	11	package org.eclipse.test.performance.ui;
12		12
		13	import java.io.PrintStream;
		14	import java.text.SimpleDateFormat;
13	import java.util.ArrayList;	15	import java.util.ArrayList;
		16	import java.util.Date;
14	import java.util.Hashtable;	17	import java.util.Hashtable;
		18	import java.util.StringTokenizer;
15		19
		20	import org.eclipse.test.internal.performance.PerformanceTestPlugin;
16	import org.eclipse.test.internal.performance.db.DB;	21	import org.eclipse.test.internal.performance.db.DB;
17	import org.eclipse.test.internal.performance.db.Scenario;	22	import org.eclipse.test.internal.performance.db.Scenario;
		23	import org.eclipse.test.internal.performance.db.SummaryEntry;
18	import org.eclipse.test.internal.performance.db.Variations;	24	import org.eclipse.test.internal.performance.db.Variations;
19		25
20	public class ScenarioStatusTable {	26	public class ScenarioStatusTable {
Lines 24-41 Link Here
24	private String scenarioPattern;	30	private String scenarioPattern;
25	private ArrayList configNames=new ArrayList();	31	private ArrayList configNames=new ArrayList();
26	private Hashtable scenarioComments;	32	private Hashtable scenarioComments;
		33	private SummaryEntry[] fingerprintEntries;
27	private final String baseline;	34	private final String baseline;
28		35
29	private class ScenarioStatus{	36	private class ScenarioStatus{
30	Hashtable statusMap;	37	Hashtable statusMap;
31	String name;	38	String name, shortName;
32	Hashtable configStatus;	39	Hashtable configStatus;
		40	Hashtable resultsMap;
33	boolean hasSlowDownExplanation=false;	41	boolean hasSlowDownExplanation=false;
		42	boolean fingerprint = false;
		43	boolean hasBaseline = true;
34		44
35	public ScenarioStatus(String scenarioName){	45	public ScenarioStatus(String scenarioName){
36	name=scenarioName;	46	name = scenarioName;
37	statusMap=new Hashtable();	47	statusMap = new Hashtable();
38	configStatus=new Hashtable();	48	configStatus = new Hashtable();
		49	resultsMap = new Hashtable();
39	}	50	}
40		51
41	}	52	}
Lines 48-207 Link Here
48	* @param scenarioPattern	59	* @param scenarioPattern
49	* @param configDescriptors	60	* @param configDescriptors
50	*/	61	*/
51	public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, String baseline){	62	public ScenarioStatusTable(Variations variations,String scenarioPattern,Hashtable configDescriptors,Hashtable scenarioComments, SummaryEntry[] fpSummaries, String baseline){
52	configMaps=configDescriptors;	63	configMaps=configDescriptors;
53	this.variations=variations;	64	this.variations=variations;
54	this.scenarioPattern=scenarioPattern;	65	this.scenarioPattern=scenarioPattern;
55	this.scenarioComments=scenarioComments;	66	this.scenarioComments=scenarioComments;
56	this.baseline= baseline;	67	this.baseline= baseline;
		68	this.fingerprintEntries = fpSummaries;
57	}	69	}
58		70
59	/**	71	/**
60	* Returns HTML representation of scenario status table.	72	* Prints the HTML representation of scenario status table into the given stream.
61	*/	73	*/
62	public String toString() {	74	public void print(PrintStream stream, boolean filter) {
63	String OS="config";	75	String OS="config";
64	String htmlTable="";	76	Scenario[] scenarios= DB.queryScenarios(variations, scenarioPattern, OS, null);
65	Scenario[] scenarios= DB.queryScenarios(variations, scenarioPattern, OS, null);	77
66		78	if (scenarios != null && scenarios.length > 0) {
67	if (scenarios != null && scenarios.length > 0) {	79	ArrayList scenarioStatusList=new ArrayList();
68	ArrayList scenarioStatusList=new ArrayList();	80
69		81	for (int i= 0; i < scenarios.length; i++) {
70	for (int i= 0; i < scenarios.length; i++) {	82	Scenario scenario= scenarios[i];
71	Scenario scenario= scenarios[i];	83	String scenarioName=scenario.getScenarioName();
72	String scenarioName=scenario.getScenarioName();	84	if (filter && !Utils.matchPattern(scenarioName, scenarioPattern)) continue;
73	// if (!Utils.matchPattern(scenarioName, scenarioPattern)) continue;	85	// returns the config names. Making assumption that indices in
74	// returns the config names. Making assumption that indices in	86	// the configs array map to the indices of the failure messages.
75	// the configs array map to the indices of the failure messages.	87	String[] configs=scenario.getTimeSeriesLabels();
76	String[] configs=scenario.getTimeSeriesLabels();	88	String[] failureMessages= scenario.getFailureMessages();
77	String[] failureMessages= scenario.getFailureMessages();	89	ScenarioStatus scenarioStatus=new ScenarioStatus(scenarioName);
78	ScenarioStatus scenarioStatus=new ScenarioStatus(scenarioName);	90	scenarioStatus.fingerprint = Utils.hasSummary(this.fingerprintEntries, scenarioName);
79	scenarioStatusList.add(scenarioStatus);
80
81	String scenarioComment= (String)scenarioComments.get(scenarioName);
82	if (scenarioComment != null)
83	scenarioStatus.hasSlowDownExplanation= true;
84
85	for (int j=0;j<configs.length;j++){
86	if (!configNames.contains(configs[j]))
87	configNames.add(configs[j]);
88
89	double[] resultStats = Utils.resultStats(variations, scenarioName, baseline, configs[j]);
90	int confidenceLevel = Utils.confidenceLevel(resultStats);
91
92	boolean hasScenarioFailure = failureMessages[j] != null && failureMessages[j].indexOf(configs[j]) != -1; // ensure correct failure message relates to config
93	StringBuffer buffer = new StringBuffer();
94	if (hasScenarioFailure) {
95	buffer.append(failureMessages[j]);
96	if (scenarioStatus.hasSlowDownExplanation) {
97	buffer.append(" - Explanation comment: ");
98	buffer.append(scenarioComment);
99	}
100	confidenceLevel \|= Utils.DEV;
101	}
102		91
103	scenarioStatus.configStatus.put(configs[j], new Integer(confidenceLevel));	92	String scenarioComment= (String)scenarioComments.get(scenarioName);
104	scenarioStatus.statusMap.put(configs[j], new Object[] { buffer.toString(), resultStats });	93	if (scenarioComment != null)
105	}	94	scenarioStatus.hasSlowDownExplanation= true;
106	}	95
107		96	int confsLength = configs.length;
108	String label=null;	97	for (int j=0; j<confsLength; j++){
109	htmlTable=htmlTable.concat("<br><h4>Scenario Status</h4>\n" +	98	if (!configNames.contains(configs[j]))
110	"The scenario status table shows all scenarios tests result for all performance test machines. It gives a complete but compact view of performance result for the component.<br>\n" +	99	configNames.add(configs[j]);
111	"For each test (ie. in each cell of this table), following information are displayed:\n" +	100
112	"<ul>\n" +	101	Variations v = (Variations) variations.clone();
113	"<li>an icon showing whether the test fails or passes and whether it's reliable or not.<br>\n" +	102	v.put(PerformanceTestPlugin.CONFIG, configs[j]);
114	"The legend for this icon is:\n" +	103	double[] resultStats = Utils.resultStats(v, scenarioName, baseline, configs[j]);
115	"<ul>\n" +	104	if (resultStats == null) continue;
116	"<li>Green (<img src=\""+Utils.OK_IMAGE+"\">): mark a <b>successful result</b>, which means this test has neither significant performance regression nor significant standard error</li>\n" +	105	if (resultStats != null && resultStats[1] < 0 && scenarioStatus.hasBaseline) scenarioStatus.hasBaseline = false;
117	"<li>Red (<img src=\""+Utils.FAIL_IMAGE+"\">): mark a <b>failing result</b>, which means this test shows a significant performance regression (more than 10%)</li>\n" +	106	int confidenceLevel = Utils.confidenceLevel(resultStats);
118	"<li>Gray (<img src=\""+Utils.FAIL_IMAGE_EXPLAINED+"\">): mark a <b>failing result</b> (see above) with a comment explaining this degradation.</li>\n" +	107
119	"<li>Yellow (<img src=\""+Utils.FAIL_IMAGE_WARN+"\"> or <img src=\""+Utils.OK_IMAGE_WARN+"\">): mark a <b>failing or successful result</b> with a significant standard error (more than "+Utils.STANDARD_ERROR_THRESHOLD_STRING+")</li>\n" +	108	boolean hasScenarioFailure = failureMessages[j] != null && failureMessages[j].indexOf(configs[j]) != -1; // ensure correct failure message relates to config
120	"<li>\"n/a\": mark a test for with <b>no</b> performance results</li>\n" +	109	StringBuffer buffer = new StringBuffer();
121	"</ul></li>\n" +	110	if (hasScenarioFailure) {
122	"<li>the value of the deviation from the baseline as a percentage (ie. formula is: <code>(build_test_time - baseline_test_time) / baseline_test_time</code>)</li>\n" +	111	buffer.append(failureMessages[j]);
123	"<li>the value of the standard error of this deviation as a percentage (ie. formula is: <code>sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time</code>)<br>\n" +	112	if (scenarioStatus.hasSlowDownExplanation) {
124	"Note that errors equal to 0 are not shown (tests which have only one iteration).</li>\n" +	113	buffer.append(" - Explanation comment: ");
125	"</ul>" +	114	buffer.append(scenarioComment);
126	"For failing tests, value of deviation with its standard error is added at the beginning of the error message you can see flying over the corresponding image.<br>\n" +	115	}
127	"Follow the link on test box corresponding image for detailed results.<br>" +	116	confidenceLevel \|= Utils.DEV;
128	"<br>\n");	117	}
129		118
130	htmlTable=htmlTable.concat("<table border=\"1\"><tr><td><h4>All "+scenarios.length+" scenarios</h4></td>\n");	119	String text = Utils.failureMessage(resultStats, false);
131	for (int i= 0; i < configNames.size(); i++){	120	if (text == null) continue;
132	label=configNames.get(i).toString();	121	scenarioStatus.configStatus.put(configs[j], new Integer(confidenceLevel));
133	String columnTitle=label;	122	scenarioStatus.statusMap.put(configs[j], buffer.toString());
134	if (configMaps!=null) {	123	scenarioStatus.resultsMap.put(configs[j], text);
135	Utils.ConfigDescriptor configDescriptor= (Utils.ConfigDescriptor)configMaps.get(label);	124	if (scenarioStatus.shortName == null) {
136	if (configDescriptor != null)	125	long time = (long) resultStats[2];
137	columnTitle=configDescriptor.description;	126	if (time == 0) { // baseline is OK
138	}	127	scenarioStatus.shortName = Utils.getScenarioShortName(scenarioName, -1);
139	htmlTable=htmlTable.concat("<td><h5>"+columnTitle +"</h5></td>");	128	} else {
140	}	129	StringBuffer shortName = new StringBuffer("*");
141		130	shortName.append(Utils.getScenarioShortName(scenarioName, -1));
142	htmlTable=htmlTable.concat("</tr>\n");	131	shortName.append(" <small>(vs. ");
143		132	if (time < 0) {
144	// counter for js class Id's	133	SimpleDateFormat dateFormat = new SimpleDateFormat("3.2_200606291905_yyyyMMddhhmm");
145	int jsIdCount=0;	134	shortName.append(dateFormat.format(new Date(-time)));
146	for (int j= 0; j < scenarioStatusList.size(); j++) {	135	} else {
147		136	SimpleDateFormat dateFormat = new SimpleDateFormat("IyyyyMMdd-hhmm");
148	ScenarioStatus status=(ScenarioStatus)scenarioStatusList.get(j);	137	shortName.append(dateFormat.format(new Date(time)));
149		138	}
150	htmlTable=htmlTable.concat("<tr><td>"+status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1)+"</td>");	139	shortName.append(")</small>");
151	for (int i=0;i<configNames.size();i++){	140	scenarioStatus.shortName = shortName.toString();
152	String message=null;	141	}
153	String configName=configNames.get(i).toString();	142	}
154	String aUrl=configName;	143
155	double[] resultStats = null;	144	if (!scenarioStatusList.contains(scenarioStatus)) {
156	if(status.statusMap.get(configName)!=null){	145	scenarioStatusList.add(scenarioStatus);
157	Object[] statusInfo = (Object[]) status.statusMap.get(configName);	146	}
158	message = (String) statusInfo[0];	147	}
159	resultStats = (double[]) statusInfo[1];	148	}
160	}	149
161		150	String label=null;
162	if (status.statusMap.containsKey(configName)){	151	stream.println("<br><h4>Scenario Status</h4>");
163	int confidence = ((Integer) status.configStatus.get(configName)).intValue();	152	stream.println("The scenario status table shows all scenarios tests result for all performance test machines.");
164	String image = Utils.getImage(confidence, resultStats, status.hasSlowDownExplanation);	153	stream.println("It gives a complete but compact view of performance result for the component.<br>");
165	StringBuffer html = new StringBuffer("\n<td><a ");	154	stream.println("For each test (ie. in each cell of this table), following information are displayed:");
166	if ((confidence & Utils.DEV) == 0 \|\| message.length() == 0){	155	stream.println("<ul>");
167	// write deviation with error in table when test pass	156	stream.println("<li>an icon showing whether the test fails or passes and whether it's reliable or not.<br>");
168	html.append("href=\"");	157	stream.println("The legend for this icon is:");
169	html.append(aUrl);	158	stream.println("<ul>");
170	html.append('/');	159	stream.print("<li>Green (<img src=\"");
171	html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));	160	stream.print(Utils.OK_IMAGE);
172	html.append(".html\">\n<img hspace=\"10\" border=\"0\" src=\"");	161	stream.print("\">): mark a <b>successful result</b>, which means this test has neither significant performance regression nor significant standard error</li>");
173	html.append(image);	162	stream.print("<li>Red (<img src=\"");
174	html.append("\"/></a>");	163	stream.print(Utils.FAIL_IMAGE);
175	} else {	164	stream.println("\">): mark a <b>failing result</b>, which means this test shows a significant performance regression (more than 10%)</li>");
176	// create message with tooltip text including deviation with error plus failure message	165	stream.print("<li>Gray (<img src=\"");
177	jsIdCount+=1;	166	stream.print(Utils.FAIL_IMAGE_EXPLAINED);
178	html.append("class=\"tooltipSource\" onMouseover=\"show_element('toolTip");	167	stream.println("\">): mark a <b>failing result</b> (see above) with a comment explaining this degradation.</li>");
179	html.append(jsIdCount);	168	stream.print("<li>Yellow (<img src=\"");
180	html.append("')\" onMouseout=\"hide_element('toolTip");	169	stream.print(Utils.FAIL_IMAGE_WARN);
181	html.append(jsIdCount);	170	stream.print("\"> or <img src=\"");
182	html.append("')\" \nhref=\"");	171	stream.print(Utils.OK_IMAGE_WARN);
183	html.append(aUrl);	172	stream.print("\">): mark a <b>failing or successful result</b> with a significant standard error (more than ");
184	html.append('/');	173	stream.print(Utils.STANDARD_ERROR_THRESHOLD_STRING);
185	html.append(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));	174	stream.println(")</li>");
186	html.append(".html\">\n<img hspace=\"10\" border=\"0\" src=\"");	175	stream.println("<li>\"n/a\": mark a test for with <b>no</b> performance results</li>");
187	html.append(image);	176	stream.println("</ul></li>");
188	html.append("\"/>\n<span class=\"hidden_tooltip\" id=\"toolTip");	177	stream.println("<li>the value of the deviation from the baseline as a percentage (ie. formula is: <code>(build_test_time - baseline_test_time) / baseline_test_time</code>)</li>");
189	html.append(jsIdCount);	178	stream.println("<li>the value of the standard error of this deviation as a percentage (ie. formula is: <code>sqrt(build_test_stddev^2 / N + baseline_test_stddev^2 / N) / baseline_test_time</code>)<br>");
190	html.append("\">");	179	stream.println("Tests with only one iteration have no available error (replaced by \"[n/a]\".</li>");
191	html.append(message);	180	stream.println("</ul>");
192	html.append("</span></a>");	181	stream.println("<u>Hints</u>:<ul>");
193	}	182	stream.println("<li>fly over image of failing tests to see the complete error message</li>");
194	html.append(Utils.failureMessage(resultStats, false));	183	stream.println("<li>to look at the complete and detailed test results, click on its image</li>");
195	html.append("</td>");	184	stream.println("<li>scenario name may be <b>emphazised</b> when the results are also displayed in the fingerprint</li>");
196	htmlTable=htmlTable.concat(html.toString());	185	stream.println("<li>scenario name may start with an '*' when the scenario has no results in the last baseline run</li>");
197	}else{	186	stream.println("<li>standard error may be '<font color=\"#CCCC00\">[n/a]</font>' when it cannot be computed, typically when the test has only one measure!</li>");
198	htmlTable=htmlTable.concat("<td>n/a</td>");	187	stream.println("</ul>");
199	}	188	stream.println();
200	}	189	stream.println("<table border=\"1\">");
201	}	190	stream.println("<tr>");
202		191	stream.print("<td><h4>All ");
203	htmlTable=htmlTable.concat("</tr>\n");	192	stream.print(scenarios.length);
204	}	193	stream.println(" scenarios</h4></td>");
205	return htmlTable;	194
206	}	195	for (int i= 0; i < configNames.size(); i++){
		196	label = configNames.get(i).toString();
		197	String columnTitle = label;
		198	if (configMaps!=null) {
		199	Utils.ConfigDescriptor configDescriptor= (Utils.ConfigDescriptor)configMaps.get(label);
		200	if (configDescriptor != null) {
		201	int idx = configDescriptor.description.indexOf('(');
		202	if (idx < 0) {
		203	columnTitle = configDescriptor.description;
		204	} else {
		205	// first line
		206	StringTokenizer tokenizer = new StringTokenizer(configDescriptor.description.substring(0, idx).trim(), " ");
		207	StringBuffer buffer = new StringBuffer(tokenizer.nextToken());
		208	while (tokenizer.hasMoreTokens()) {
		209	buffer.append(" ");
		210	buffer.append(tokenizer.nextToken());
		211	}
		212	buffer.append(' ');
		213	// second line
		214	tokenizer = new StringTokenizer(configDescriptor.description.substring(idx).trim(), " ");
		215	buffer.append(tokenizer.nextToken());
		216	while (tokenizer.hasMoreTokens()) {
		217	buffer.append(" ");
		218	buffer.append(tokenizer.nextToken());
		219	}
		220	columnTitle = buffer.toString();
		221	}
		222	}
		223	}
		224	stream.print("<td><h5>");
		225	stream.print(columnTitle);
		226	stream.println("</h5>");
		227	}
		228
		229	// counter for js class Id's
		230	int jsIdCount=0;
		231	for (int j= 0; j < scenarioStatusList.size(); j++) {
		232
		233	ScenarioStatus status=(ScenarioStatus)scenarioStatusList.get(j);
		234
		235	stream.println("<tr>");
		236	stream.print("<td>");
		237	if (status.fingerprint) stream.print("<b>");
		238	if (!status.hasBaseline) stream.print("*");
		239	// stream.print(status.name.substring(status.name.indexOf(".",status.name.indexOf(".test")+1)+1));
		240	stream.print(status.shortName);
		241	if (!status.hasBaseline) stream.print("</i>");
		242	if (status.fingerprint) stream.print("</b>");
		243	stream.println();
		244	for (int i=0;i<configNames.size();i++){
		245	String configName=configNames.get(i).toString();
		246	String aUrl=configName;
		247	if (status.statusMap.containsKey(configName)){
		248	String message = (String) status.statusMap.get(configName);
		249	int confidence = ((Integer) status.configStatus.get(configName)).intValue();
		250	String image = Utils.getImage(confidence, status.hasSlowDownExplanation);
		251	stream.print("<td><a ");
		252	if ((confidence & Utils.DEV) == 0 \|\| message.length() == 0){
		253	// write deviation with error in table when test pass
		254	stream.print("href=\"");
		255	stream.print(aUrl);
		256	stream.print('/');
		257	stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
		258	stream.println(".html\">");
		259	stream.print("<img hspace=\"10\" border=\"0\" src=\"");
260	stream.print(image);
261	stream.println("\"/></a>");
262	} else {
263	// create message with tooltip text including deviation with error plus failure message
264	jsIdCount+=1;
265	stream.print("class=\"tooltipSource\" onMouseover=\"show_element('toolTip");
266	stream.print(jsIdCount);
267	stream.print("')\" onMouseout=\"hide_element('toolTip");
268	stream.print(jsIdCount);
269	stream.print("')\" \nhref=\"");
270	stream.print(aUrl);
271	stream.print('/');
272	stream.print(status.name.replace('#', '.').replace(':', '_').replace('\\', '_'));
273	stream.println(".html\">");
274	stream.print("<img hspace=\"10\" border=\"0\" src=\"");
275	stream.print(image);
276	stream.println("\"/>");
277	stream.print("<span class=\"hidden_tooltip\" id=\"toolTip");
278	stream.print(jsIdCount);
279	stream.print("\">");
280	stream.print(message);
281	stream.println("</span></a>");
282	}
283	String results = (String) status.resultsMap.get(configName);
284	if (results != null) stream.println(results);
285	}else{
286	stream.println("<td>n/a");
287	}
288	}
289	stream.flush();
290	}
291	stream.append("</table>");
292	}
293	}
207	}	294	}

Lines 19-35 Link Here

(-)src/org/eclipse/test/performance/ui/FingerPrint.java (-26 / +38 lines)
19	import java.util.Hashtable;	19	import java.util.Hashtable;
20		20
21	import org.eclipse.swt.SWT;	21	import org.eclipse.swt.SWT;
22	import org.eclipse.swt.events.PaintEvent;
23	import org.eclipse.swt.events.PaintListener;
24	import org.eclipse.swt.graphics.GC;	22	import org.eclipse.swt.graphics.GC;
25	import org.eclipse.swt.graphics.Image;	23	import org.eclipse.swt.graphics.Image;
26	import org.eclipse.swt.graphics.ImageData;	24	import org.eclipse.swt.graphics.ImageData;
27	import org.eclipse.swt.graphics.ImageLoader;	25	import org.eclipse.swt.graphics.ImageLoader;
28	import org.eclipse.swt.graphics.Point;
29	import org.eclipse.swt.layout.FillLayout;
30	import org.eclipse.swt.widgets.Canvas;
31	import org.eclipse.swt.widgets.Display;	26	import org.eclipse.swt.widgets.Display;
32	import org.eclipse.swt.widgets.Shell;
33	import org.eclipse.test.internal.performance.PerformanceTestPlugin;	27	import org.eclipse.test.internal.performance.PerformanceTestPlugin;
34	import org.eclipse.test.internal.performance.data.Dim;	28	import org.eclipse.test.internal.performance.data.Dim;
35	import org.eclipse.test.internal.performance.db.DB;	29	import org.eclipse.test.internal.performance.db.DB;
Lines 37-45 Link Here
37	import org.eclipse.test.internal.performance.db.SummaryEntry;	31	import org.eclipse.test.internal.performance.db.SummaryEntry;
38	import org.eclipse.test.internal.performance.db.TimeSeries;	32	import org.eclipse.test.internal.performance.db.TimeSeries;
39	import org.eclipse.test.internal.performance.db.Variations;	33	import org.eclipse.test.internal.performance.db.Variations;
40	import org.eclipse.test.internal.performance.eval.StatisticsUtil;
41	import org.eclipse.test.internal.performance.eval.StatisticsUtil.Percentile;
42	import org.eclipse.test.performance.Dimension;
43	import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;	34	import org.eclipse.test.performance.ui.Utils.ConfigDescriptor;
44		35
45		36
Lines 74-90 Link Here
74	if (component==null){	65	if (component==null){
75	entries= DB.querySummaries(variations,null);	66	entries= DB.querySummaries(variations,null);
76	this.component="";	67	this.component="";
77	}	68	} else {
78	else
79	entries=DB.querySummaries(variations,component+'%');	69	entries=DB.querySummaries(variations,component+'%');
80	run(entries);	70	}
		71	run();
81	}	72	}
82		73
83	/**	74	/**
84	* Creates the fingerprint gif, image map and scenario status table for the component.	75	* Creates the fingerprint gif, image map and scenario status table for the component.
85	* @param entries - the result of a database query for summaries for a specified variation.
86	*/	76	*/
87	public void run(Object[] entries) {	77	public void run() {
88	new File(outputDirectory).mkdirs();	78	new File(outputDirectory).mkdirs();
89	String referenceName=referenceBuildId;	79	String referenceName=referenceBuildId;
90	String currentName=currentBuildId;	80	String currentName=currentBuildId;
Lines 101-112 Link Here
101		91
102	if (entries != null) {	92	if (entries != null) {
103	for (int i= 0; i < entries.length; i++) {	93	for (int i= 0; i < entries.length; i++) {
104	SummaryEntry se= (SummaryEntry)entries[i];	94	SummaryEntry summary = entries[i];
105	if (se.comment==null)	95	if (summary.comment==null)
106	add(bar, se.shortName, new Dim[] { se.dimension }, se.scenarioName);	96	add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName);
107	else{	97	else{
108	setComment(se.scenarioName, se.comment);	98	setComment(summary.scenarioName, summary.comment);
109	add(bar, se.shortName, new Dim[] { se.dimension }, se.scenarioName,se.comment);	99	add(summary.shortName, new Dim[] { summary.dimension }, summary.scenarioName, summary.comment);
110	}	100	}
111	}	101	}
112	}	102	}
Lines 115-121 Link Here
115		105
116	if (component=="")	106	if (component=="")
117	outName= "FP_"+referenceName + '_' + currentBuildId+"."+configDescriptor.name;	107	outName= "FP_"+referenceName + '_' + currentBuildId+"."+configDescriptor.name;
118	save(bar, outputDirectory + '/' + outName);	108	save(outputDirectory + '/' + outName);
119		109
120	//show(bar);	110	//show(bar);
121		111
Lines 127-145 Link Here
127	scenarioComments.put(scenario,comment);	117	scenarioComments.put(scenario,comment);
128	}	118	}
129		119
130	private void add(BarGraph bar, String name, Dim[] dims, String scenarioName) {	120	private void add(String name, Dim[] dims, String scenarioName) {
131	add (bar,name,dims,scenarioName,null);	121	add (name,dims,scenarioName,null);
132	}	122	}
133		123
134	private void add(BarGraph bar, String name, Dim[] dims, String scenarioName,String comment) {	124	private void add(String name, Dim[] dims, String scenarioName, String comment) {
135	String refData= "";	125	String refData= "";
136	Scenario scenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, referenceBuildId, currentBuildId, dims);	126	Scenario scenario= DB.getScenarioSeries(scenarioName, variations, PerformanceTestPlugin.BUILD, referenceBuildId, currentBuildId, dims);
137	String[] timeSeriesLabels= scenario.getTimeSeriesLabels();	127	String[] timeSeriesLabels= scenario.getTimeSeriesLabels();
		128	long buildTime = 0;
138	if (timeSeriesLabels.length == 2) {	129	if (timeSeriesLabels.length == 2) {
139	// we mark the label with a '*' or '' to indicate that no data was available for the specified builds	130	// we mark the label with a '*' or '' to indicate that no data was available for the specified builds
140	if (!timeSeriesLabels[0].equals(referenceBuildId)) {	131	if (!timeSeriesLabels[0].equals(referenceBuildId)) {
141	name= '*' + name;	132	name= '*' + name;
142	refData= " (" + timeSeriesLabels[0] + ")";	133	refData= " (" + timeSeriesLabels[0] + ")";
		134	buildTime = Utils.getBuildTime(timeSeriesLabels[0]);
143	} else if (!timeSeriesLabels[1].equals(currentBuildId)) {	135	} else if (!timeSeriesLabels[1].equals(currentBuildId)) {
144	name= '' + name;	136	name= '' + name;
145	refData= " (" + timeSeriesLabels[1] + ")";	137	refData= " (" + timeSeriesLabels[1] + ")";
Lines 153-159 Link Here
153	double percent= 0.0;	145	double percent= 0.0;
154	boolean hasConfidentResult= true;	146	boolean hasConfidentResult= true;
155	if (l > 1) {	147	if (l > 1) {
156	hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name);	148	// hasConfidentResult= Utils.hasConfidentResult(variations, scenario.getScenarioName(),referenceBuildId,configDescriptor.name);
		149	hasConfidentResult= Utils.hasConfidentResult(timeSeries, buildTime);
157	/*if (!rejectNullHypothesis) {	150	/*if (!rejectNullHypothesis) {
158	NumberFormat percentFormatter= NumberFormat.getPercentInstance();	151	NumberFormat percentFormatter= NumberFormat.getPercentInstance();
159	String statisticsComment= "There is not enough evidence to reject the null hypothesis at the " + percentFormatter.format(percentile.inside()) + "level";	152	String statisticsComment= "There is not enough evidence to reject the null hypothesis at the " + percentFormatter.format(percentile.inside()) + "level";
Lines 178-184 Link Here
178		171
179	}	172	}
180		173
181	private void save(BarGraph bar, String output) {	174	private void save(String output) {
182		175
183	//if (bar.getFItems().size()==0)	176	//if (bar.getFItems().size()==0)
184	//return;	177	//return;
Lines 217-223 Link Here
217		210
218	/*	211	/*
219	* Displays bar graph in window	212	* Displays bar graph in window
220	*/	213	*
221	private void show(final BarGraph bar) {	214	private void show(final BarGraph bar) {
222	Display display= new Display();	215	Display display= new Display();
223		216
Lines 246-249 Link Here
246	public String getOutName() {	239	public String getOutName() {
247	return outName;	240	return outName;
248	}	241	}
		242	*/
		243
		244	/**
		245	* @return - an html representation of the fingerprint.
		246	*/
		247	public String getImageMap() {
		248	String componentDescription = this.configDescriptor.description;
		249	String areas = this.bar.getAreas();
		250	if (areas == null)
		251	areas = "";
		252	String output = "";
		253	if (new File(this.outputDirectory, this.outName + ".gif").exists()) {
		254	output = "<h4>" + componentDescription + "</h4>";
		255	output = output.concat("<img src=\"" + this.outName + ".gif\" usemap=\"#" + this.outName + "\">" + "<map name=\"" + this.outName + "\">" + areas + "</map>\n");
		256	} else {
		257	output = output.concat("<br><br>There is no fingerprint for " + componentDescription + "<br><br>\n");
		258	}
		259	return output;
		260	}
249	}	261	}

Added Link Here

(-)src/org/eclipse/test/performance/ui/Messages.java (+75 lines)
1	/*******************************************************************************
2	* Copyright (c) 2000, 2007 IBM Corporation and others.
3	* All rights reserved. This program and the accompanying materials
4	* are made available under the terms of the Eclipse Public License v1.0
5	* which accompanies this distribution, and is available at
6	* http://www.eclipse.org/legal/epl-v10.html
7	*
8	* Contributors:
9	* IBM Corporation - initial API and implementation
10	*******************************************************************************/
11	package org.eclipse.test.performance.ui;
12
13	import java.text.MessageFormat;
14
15	import org.eclipse.osgi.util.NLS;
16
17	public class Messages extends NLS {
18
19	private static final String BUNDLE_NAME = "org.eclipse.test.performance.ui.messages";//$NON-NLS-1$
20
21	private Messages() {
22	// Do not instantiate
23	}
24
25	public static String standardError;
26
27	static {
28	NLS.initializeMessages(BUNDLE_NAME, Messages.class);
29	}
30
31	/**
32	* Bind the given message's substitution locations with the given string values.
33	*
34	* @param message the message to be manipulated
35	* @return the manipulated String
36	*/
37	public static String bind(String message) {
38	return bind(message, null);
39	}
40
41	/**
42	* Bind the given message's substitution locations with the given string values.
43	*
44	* @param message the message to be manipulated
45	* @param binding the object to be inserted into the message
46	* @return the manipulated String
47	*/
48	public static String bind(String message, Object binding) {
49	return bind(message, new Object[] {binding});
50	}
51
52	/**
53	* Bind the given message's substitution locations with the given string values.
54	*
55	* @param message the message to be manipulated
56	* @param binding1 An object to be inserted into the message
57	* @param binding2 A second object to be inserted into the message
58	* @return the manipulated String
59	*/
60	public static String bind(String message, Object binding1, Object binding2) {
61	return bind(message, new Object[] {binding1, binding2});
62	}
63
64	/**
65	* Bind the given message's substitution locations with the given string values.
66	*
67	* @param message the message to be manipulated
68	* @param bindings An array of objects to be inserted into the message
69	* @return the manipulated String
70	*/
71	public static String bind(String message, Object[] bindings) {
72	return MessageFormat.format(message, bindings);
73	}
74
75	}

Added Link Here

(-)src/org/eclipse/test/performance/ui/messages.properties (+14 lines)
1	###############################################################################
2	# Copyright (c) 2000, 2007 IBM Corporation and others.
3	# All rights reserved. This program and the accompanying materials
4	# are made available under the terms of the Eclipse Public License v1.0
5	# which accompanies this distribution, and is available at
6	# http://www.eclipse.org/legal/epl-v10.html
7	#
8	# Contributors:
9	# IBM Corporation - initial API and implementation
10	###############################################################################
11
12	### Performance Tests Messages.
13
14	standardError = Standard error on this test is {0} (should be less than {1} to become reliable!)