Attachment 194413 Details for Bug 126121 – Stub to parallelize project build

[patch] Stub to parallelize project build

parallelPatch.txt (text/plain), 19.71 KB, created by Jens Kübler

on 2011-04-29 17:47:34 EDT

(hide)

Description:

Filename:

MIME Type:

Creator: Jens Kübler

Created: 2011-04-29 17:47:34 EDT

Size: 19.71 KB

patch

obsolete

>### Eclipse Workspace Patch 1.0
>#P org.eclipse.core.resources
>Index: src/org/eclipse/core/internal/events/BuildManager.java
>===================================================================
>RCS file: /cvsroot/eclipse/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildManager.java,v
>retrieving revision 1.117
>diff -u -r1.117 BuildManager.java
>--- src/org/eclipse/core/internal/events/BuildManager.java	22 Apr 2010 09:03:35 -0000	1.117
>+++ src/org/eclipse/core/internal/events/BuildManager.java	29 Apr 2011 21:33:38 -0000
>@@ -13,6 +13,8 @@
>  *******************************************************************************/
> package org.eclipse.core.internal.events;
> 
>+import org.eclipse.core.resources.IncrementalProjectBuilder;
>+
> import java.util.*;
> import org.eclipse.core.internal.dtree.DeltaDataTree;
> import org.eclipse.core.internal.resources.*;
>@@ -132,19 +134,19 @@
> 
> 	private void basicBuild(int trigger, IncrementalProjectBuilder builder, Map args, MultiStatus status, IProgressMonitor monitor) {
> 		try {
>-			currentBuilder = builder;
>+			InternalBuilder currentBuilder2 = builder;
> 			//clear any old requests to forget built state
>-			currentBuilder.clearForgetLastBuiltState();
>+			currentBuilder2.clearForgetLastBuiltState();
> 			// Figure out want kind of build is needed
> 			boolean clean = trigger == IncrementalProjectBuilder.CLEAN_BUILD;
>-			currentLastBuiltTree = currentBuilder.getLastBuiltTree();
>+			currentLastBuiltTree = currentBuilder2.getLastBuiltTree();
> 			// If no tree is available we have to do a full build
> 			if (!clean && currentLastBuiltTree == null)
> 				trigger = IncrementalProjectBuilder.FULL_BUILD;
> 			//don't build if this builder doesn't respond to the given trigger
> 			if (!builder.getCommand().isBuilding(trigger)) {
> 				if (clean)
>-					currentBuilder.setLastBuiltTree(null);
>+					currentBuilder2.setLastBuiltTree(null);
> 				return;
> 			}
> 			// For incremental builds, grab a pointer to the current state before computing the delta
>@@ -152,13 +154,13 @@
> 			int depth = -1;
> 			try {
> 				//short-circuit if none of the projects this builder cares about have changed.
>-				if (!needsBuild(currentBuilder, trigger)) {
>+				if (!needsBuild(currentBuilder2, trigger)) {
> 					//use up the progress allocated for this builder
> 					monitor.beginTask("", 1); //$NON-NLS-1$
> 					monitor.done();
> 					return;
> 				}
>-				String name = currentBuilder.getLabel();
>+				String name = currentBuilder2.getLabel();
> 				String message;
> 				if (name != null)
> 					message = NLS.bind(Messages.events_invoking_2, name, builder.getProject().getFullPath());
>@@ -169,23 +171,23 @@
> 				//release workspace lock while calling builders
> 				depth = getWorkManager().beginUnprotected();
> 				//do the build
>-				SafeRunner.run(getSafeRunnable(trigger, args, status, monitor));
>+				SafeRunner.run(getSafeRunnable(trigger, args, status, monitor, currentBuilder2));
> 			} finally {
> 				if (depth >= 0)
> 					getWorkManager().endUnprotected(depth);
> 				// Be sure to clean up after ourselves.
>-				if (clean || currentBuilder.wasForgetStateRequested()) {
>-					currentBuilder.setLastBuiltTree(null);
>+				if (clean || currentBuilder2.wasForgetStateRequested()) {
>+					currentBuilder2.setLastBuiltTree(null);
> 				} else {
> 					// remember the current state as the last built state.
> 					ElementTree lastTree = workspace.getElementTree();
> 					lastTree.immutable();
>-					currentBuilder.setLastBuiltTree(lastTree);
>+					currentBuilder2.setLastBuiltTree(lastTree);
> 				}
> 				hookEndBuild(builder);
> 			}
> 		} finally {
>-			currentBuilder = null;
>+//			currentBuilder = null;
> 			currentTree = null;
> 			currentLastBuiltTree = null;
> 			currentDelta = null;
>@@ -198,7 +200,12 @@
> 				checkCanceled(trigger, monitor);
> 				BuildCommand command = (BuildCommand) commands[i];
> 				IProgressMonitor sub = Policy.subMonitorFor(monitor, 1);
>-				IncrementalProjectBuilder builder = getBuilder(project, command, i, status);
>+				IncrementalProjectBuilder builder;
>+				if(IncrementalProjectBuilder.PARALLEL_FULL_BUILD == trigger) {
>+					builder = getParallelBuilder(project, command, i, status);
>+				} else {
>+					builder = getBuilder(project, command, i, status);
>+				}
> 				if (builder != null)
> 					basicBuild(trigger, builder, command.getArguments(false), status, sub);
> 			}
>@@ -207,6 +214,23 @@
> 		}
> 	}
> 
>+	private IncrementalProjectBuilder getParallelBuilder(IProject project, BuildCommand command, int buildSpecIndex, MultiStatus status) throws CoreException {
>+		// always construct a new builder for parallel builds (think about pooling here)
>+		InternalBuilder result = initializeBuilder(command.getBuilderName(), project, buildSpecIndex, status);
>+		((BuildCommand) command).setBuilder((IncrementalProjectBuilder) result);
>+		result.setCommand(command);
>+		result.setProject(project);
>+		result.startupOnInitialize();
>+//		}
>+		if (!validateNature(result, command.getBuilderName())) {
>+			//skip this builder and null its last built tree because it is invalid
>+			//if the nature gets added or re-enabled a full build will be triggered
>+			result.setLastBuiltTree(null);
>+			return null;
>+		}
>+		return (IncrementalProjectBuilder) result;
>+	}
>+
> 	/**
> 	 * Runs all builders on the given project. 
> 	 * @return A status indicating if the build succeeded or failed
>@@ -327,7 +351,7 @@
> 	 * Runs all builders on all projects. 
> 	 * @return A status indicating if the build succeeded or failed
> 	 */
>-	public IStatus build(int trigger, IProgressMonitor monitor) {
>+	public IStatus build(final int trigger, IProgressMonitor monitor) {
> 		monitor = Policy.monitorFor(monitor);
> 		try {
> 			monitor.beginTask(Messages.events_building_0, TOTAL_BUILD_WORK);
>@@ -335,12 +359,12 @@
> 				return Status.OK_STATUS;
> 			try {
> 				hookStartBuild(trigger);
>-				IProject[] ordered = workspace.getBuildOrder();
>-				HashSet leftover = new HashSet(Arrays.asList(workspace.getRoot().getProjects(IContainer.INCLUDE_HIDDEN)));
>-				leftover.removeAll(Arrays.asList(ordered));
>-				IProject[] unordered = (IProject[]) leftover.toArray(new IProject[leftover.size()]);
>-				MultiStatus status = new MultiStatus(ResourcesPlugin.PI_RESOURCES, IResourceStatus.BUILD_FAILED, Messages.events_errors, null);
>-				basicBuildLoop(ordered, unordered, trigger, status, monitor);
>+				final MultiStatus status = new MultiStatus(ResourcesPlugin.PI_RESOURCES, IResourceStatus.BUILD_FAILED, Messages.events_errors, null);
>+					IProject[] ordered = workspace.getBuildOrder();
>+					HashSet leftover = new HashSet(Arrays.asList(workspace.getRoot().getProjects(IContainer.INCLUDE_HIDDEN)));
>+					leftover.removeAll(Arrays.asList(ordered));
>+					IProject[] unordered = (IProject[]) leftover.toArray(new IProject[leftover.size()]);
>+					basicBuildLoop(ordered, unordered, trigger, status, monitor);
> 				return status;
> 			} finally {
> 				hookEndBuild(trigger);
>@@ -455,7 +479,7 @@
> 	 * The outermost workspace operation has finished.  Do an autobuild if necessary.
> 	 */
> 	public void endTopLevel(boolean needsBuild) {
>-		autoBuildJob.build(needsBuild);
>+//		autoBuildJob.build(needsBuild);
> 	}
> 
> 	/**
>@@ -594,8 +618,9 @@
> 
> 	/**
> 	 * Returns the safe runnable instance for invoking a builder
>+	 * @param currentBuilder2 
> 	 */
>-	private ISafeRunnable getSafeRunnable(final int trigger, final Map args, final MultiStatus status, final IProgressMonitor monitor) {
>+	private ISafeRunnable getSafeRunnable(final int trigger, final Map args, final MultiStatus status, final IProgressMonitor monitor, final InternalBuilder currentBuilder2) {
> 		return new ISafeRunnable() {
> 			public void handleException(Throwable e) {
> 				if (e instanceof OperationCanceledException) {
>@@ -603,18 +628,18 @@
> 						Policy.debug("Build canceled"); //$NON-NLS-1$
> 					//just discard built state when a builder cancels, to ensure
> 					//that it is called again on the very next build.
>-					currentBuilder.forgetLastBuiltState();
>+					currentBuilder2.forgetLastBuiltState();
> 					throw (OperationCanceledException) e;
> 				}
> 				//ResourceStats.buildException(e);
> 				// don't log the exception....it is already being logged in SafeRunner#run
> 
> 				//add a generic message to the MultiStatus
>-				String builderName = currentBuilder.getLabel();
>+				String builderName = currentBuilder2.getLabel();
> 				if (builderName == null || builderName.length() == 0)
>-					builderName = currentBuilder.getClass().getName();
>-				String pluginId = currentBuilder.getPluginId();
>-				String message = NLS.bind(Messages.events_builderError, builderName, currentBuilder.getProject().getName());
>+					builderName = currentBuilder2.getClass().getName();
>+				String pluginId = currentBuilder2.getPluginId();
>+				String message = NLS.bind(Messages.events_builderError, builderName, currentBuilder2.getProject().getName());
> 				status.add(new Status(IStatus.ERROR, pluginId, IResourceStatus.BUILD_FAILED, message, e));
> 
> 				//add the exception status to the MultiStatus
>@@ -626,12 +651,12 @@
> 				IProject[] prereqs = null;
> 				//invoke the appropriate build method depending on the trigger
> 				if (trigger != IncrementalProjectBuilder.CLEAN_BUILD)
>-					prereqs = currentBuilder.build(trigger, args, monitor);
>+					prereqs = currentBuilder2.build(trigger, args, monitor);
> 				else
>-					currentBuilder.clean(monitor);
>+					currentBuilder2.clean(monitor);
> 				if (prereqs == null)
> 					prereqs = new IProject[0];
>-				currentBuilder.setInterestingProjects((IProject[]) prereqs.clone());
>+				currentBuilder2.setInterestingProjects((IProject[]) prereqs.clone());
> 			}
> 		};
> 	}
>@@ -1032,4 +1057,37 @@
> 			Policy.log(status);
> 		return workspace.getRoot();
> 	}
>+
>+	public void buildParallel(IProgressMonitor subMonitorFor) {
>+			IProject[] projects = workspace.getRoot().getProjects(IContainer.INCLUDE_HIDDEN);
>+			DirectedGraphs<IProject> directedGraphs = new DirectedGraphs<IProject>();
>+			for (IProject project : projects) {
>+				directedGraphs.addNode(new Node<IProject>(project));
>+			}
>+			for (IProject project : projects) {
>+				if (!project.isAccessible())
>+					continue;
>+				ProjectDescription desc = ((Project)project).internalGetDescription();
>+				if (desc == null)
>+					continue;
>+				IProject[] refs = desc.getAllReferences(false);
>+				Node<IProject> node = directedGraphs.getNodeWithContent(project);
>+				for (IProject referencedProject : refs) {
>+					Node<IProject> node2 = directedGraphs.getNodeWithContent(referencedProject);
>+					node2.addOutgoingEdge(node);
>+				}
>+			}
>+			ThreadPoolBuilder threadPoolBuilder = new ThreadPoolBuilder();
>+			try {
>+				final MultiStatus status = new MultiStatus(ResourcesPlugin.PI_RESOURCES, IResourceStatus.BUILD_FAILED, Messages.events_errors, null);
>+				IBuildExecutor<IProject> buildExecutor = new IBuildExecutor<IProject>() {
>+				public void executeBuild(IProject project) {
>+					NullProgressMonitor progressMonitor = new NullProgressMonitor();
>+					basicBuild(project, IncrementalProjectBuilder.PARALLEL_FULL_BUILD, status, progressMonitor);
>+				}};
>+				threadPoolBuilder.run(directedGraphs, buildExecutor);
>+			} catch (InterruptedException e) {
>+				e.printStackTrace();
>+			}
>+	}
> }
>Index: src/org/eclipse/core/internal/events/BuildRunnable.java
>===================================================================
>RCS file: src/org/eclipse/core/internal/events/BuildRunnable.java
>diff -N src/org/eclipse/core/internal/events/BuildRunnable.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ src/org/eclipse/core/internal/events/BuildRunnable.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,31 @@
>+package org.eclipse.core.internal.events;
>+
>+import java.util.ArrayList;
>+import java.util.List;
>+import java.util.concurrent.BlockingQueue;
>+
>+public final class BuildRunnable<T> implements Runnable {
>+	
>+	private final Node<T> currentNode;
>+	private final BlockingQueue queue;
>+	private final IBuildExecutor<T> executor;
>+
>+	protected BuildRunnable(Node<T> currentNode, BlockingQueue queue, IBuildExecutor<T> executor) {
>+		this.currentNode = currentNode;
>+		this.queue = queue;
>+		this.executor = executor;
>+	}
>+	
>+	public void run() {
>+			System.out.println("start compile " + currentNode.getContent()); //$NON-NLS-1$
>+			executor.executeBuild(currentNode.getContent());
>+			System.out.println("end compile " + currentNode.getContent()); //$NON-NLS-1$
>+			List<Node<T>> outgoingNodes = new ArrayList<Node<T>>(currentNode.getOutgoingEdges()); 
>+			currentNode.removeAllOutgoingEdges();
>+				for (Node<T> node : outgoingNodes) {
>+					if(node.getIncomingEdges().isEmpty()) {
>+						queue.add(node);
>+					}
>+				}
>+	}
>+}
>\ No newline at end of file
>Index: src/org/eclipse/core/internal/events/DirectedGraphs.java
>===================================================================
>RCS file: src/org/eclipse/core/internal/events/DirectedGraphs.java
>diff -N src/org/eclipse/core/internal/events/DirectedGraphs.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ src/org/eclipse/core/internal/events/DirectedGraphs.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,58 @@
>+package org.eclipse.core.internal.events;
>+
>+import java.util.Collection;
>+import java.util.Collections;
>+import java.util.HashSet;
>+import java.util.LinkedHashSet;
>+import java.util.Set;
>+
>+/**
>+ * A directed graph holds all nodes. Nodes need not be connected which is why a container object is required
>+ * 
>+ * @author jens
>+ *
>+ */
>+public class DirectedGraphs<T> {
>+	
>+	private Set<Node<T>> nodes;
>+
>+	public DirectedGraphs() {
>+		nodes = Collections.synchronizedSet(new HashSet<Node<T>>());
>+	}
>+	
>+	public void addNode(Node<T> node) {
>+		nodes.add(node);
>+	}
>+
>+	public void removeNode(Node<T> node) {
>+		nodes.remove(node);
>+	}
>+	
>+	public void addAll(Collection<Node<T>> nodesToBeAdded) {
>+		nodes.addAll(nodesToBeAdded);
>+	}
>+	
>+	public Node<T> getNodeWithContent(T content) {
>+		for (Node<T> node : nodes) {
>+			if(content.equals(node.getContent())) {
>+				return node;
>+			}
>+		}
>+		return null;
>+	}
>+	
>+	public synchronized Set<Node<T>> getRootNodes() {
>+		Set<Node<T>> set = new LinkedHashSet<Node<T>>();
>+		for (Node<T> node : nodes) {
>+			if(node.getIncomingEdges().isEmpty()) {
>+				set.add(node);
>+			}
>+		}
>+		return set;
>+	}
>+	
>+	public boolean isEmtpy() {
>+		return nodes.isEmpty();
>+	}
>+
>+}
>Index: src/org/eclipse/core/internal/events/IBuildExecutor.java
>===================================================================
>RCS file: src/org/eclipse/core/internal/events/IBuildExecutor.java
>diff -N src/org/eclipse/core/internal/events/IBuildExecutor.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ src/org/eclipse/core/internal/events/IBuildExecutor.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,21 @@
>+/*******************************************************************************
>+ * Copyright (c) 2011 IBM Corporation and others.
>+ * All rights reserved. This program and the accompanying materials
>+ * are made available under the terms of the Eclipse Public License v1.0
>+ * which accompanies this distribution, and is available at
>+ * http://www.eclipse.org/legal/epl-v10.html
>+ * 
>+ * Contributors:
>+ *     IBM Corporation - initial API and implementation
>+ *******************************************************************************/
>+package org.eclipse.core.internal.events;
>+
>+
>+/**
>+ * 
>+ */
>+public interface IBuildExecutor<T> {
>+	
>+	public void executeBuild(T content);
>+
>+}
>Index: src/org/eclipse/core/internal/events/Node.java
>===================================================================
>RCS file: src/org/eclipse/core/internal/events/Node.java
>diff -N src/org/eclipse/core/internal/events/Node.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ src/org/eclipse/core/internal/events/Node.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,129 @@
>+package org.eclipse.core.internal.events;
>+
>+import java.text.MessageFormat;
>+
>+import java.util.Collections;
>+import java.util.Iterator;
>+import java.util.LinkedHashSet;
>+import java.util.Set;
>+
>+/**
>+ * A simple node object having references to other node objects 
>+ *
>+ * @param <T>
>+ */
>+public class Node<T> {
>+	
>+	private Set<Node<T>> incomingEdges;
>+
>+	private Set<Node<T>> outgoingEdges;
>+	
>+	private T content;
>+	
>+	public Node(T content) {
>+		this.content = content;
>+		incomingEdges = Collections.synchronizedSet(new LinkedHashSet<Node<T>>()); 
>+		outgoingEdges = Collections.synchronizedSet(new LinkedHashSet<Node<T>>()); 
>+	}
>+	
>+	public T getContent() {
>+		return content;
>+	}
>+
>+	public void setContent(T content) {
>+		this.content = content;
>+	}
>+
>+	public synchronized Set<Node<T>> getIncomingEdges() {
>+		return incomingEdges;
>+	}
>+
>+	public synchronized Set<Node<T>> getOutgoingEdges() {
>+		return outgoingEdges;
>+	}
>+
>+	public synchronized void addIncomingEdge(Node<T> node) {
>+		if(!incomingEdges.contains(node)) {
>+			incomingEdges.add(node);
>+			node.addOutgoingEdge(this);
>+		}
>+	}
>+
>+	public synchronized void addOutgoingEdge(Node<T> node) {
>+		if(!outgoingEdges.contains(node)) {
>+			outgoingEdges.add(node);
>+			node.addIncomingEdge(this);
>+		}
>+	}
>+	
>+	public synchronized void removeOutgoingEdge(Node<T> node) {
>+		if(outgoingEdges.contains(node)) {
>+			outgoingEdges.remove(node);
>+			node.removeIncomingEdge(this);
>+		}
>+	}
>+	
>+	public synchronized void removeIncomingEdge(Node<T> node) {
>+		if(incomingEdges.contains(node)) {
>+			incomingEdges.remove(node);
>+			node.removeOutgoingEdge(this);
>+		}
>+	}
>+
>+	public synchronized void removeAllOutgoingEdges() {
>+		for (Iterator<Node<T>> iterator = outgoingEdges.iterator(); iterator.hasNext();) {
>+			Node<T> node = iterator.next();
>+			node.internalRemoveIncomingEdge(this);
>+			iterator.remove();
>+		}
>+	}
>+
>+	public synchronized void removeAllIncomingEdges() {
>+		for (Iterator<Node<T>> iterator = incomingEdges.iterator(); iterator.hasNext();) {
>+			Node<T> node = iterator.next();
>+			node.internalRemoveOutgoingEdge(this);
>+			iterator.remove();
>+		}
>+	}
>+
>+	synchronized void internalRemoveOutgoingEdge(Node<T> node) {
>+		outgoingEdges.remove(node);
>+	}
>+
>+	synchronized void internalRemoveIncomingEdge(Node<T> node) {
>+		incomingEdges.remove(node);
>+	}
>+
>+	@Override
>+	public int hashCode() {
>+		final int prime = 31;
>+		int result = 1;
>+		result = prime * result + ((content == null) ? 0 : content.hashCode());
>+		return result;
>+	}
>+
>+	@Override
>+	public boolean equals(Object obj) {
>+		if (this == obj)
>+			return true;
>+		if (obj == null)
>+			return false;
>+		if (getClass() != obj.getClass())
>+			return false;
>+		Node other = (Node) obj;
>+		if (content == null) {
>+			if (other.content != null)
>+				return false;
>+		} else if (!content.equals(other.content))
>+			return false;
>+		return true;
>+	}
>+
>+	@Override
>+	public String toString() {
>+		return MessageFormat.format("Node [content={0}]", content); //$NON-NLS-1$
>+	}
>+
>+	
>+	
>+}
>Index: src/org/eclipse/core/internal/events/ThreadPoolBuilder.java
>===================================================================
>RCS file: src/org/eclipse/core/internal/events/ThreadPoolBuilder.java
>diff -N src/org/eclipse/core/internal/events/ThreadPoolBuilder.java
>--- /dev/null	1 Jan 1970 00:00:00 -0000
>+++ src/org/eclipse/core/internal/events/ThreadPoolBuilder.java	1 Jan 1970 00:00:00 -0000
>@@ -0,0 +1,31 @@
>+package org.eclipse.core.internal.events;
>+
>+import java.util.concurrent.ArrayBlockingQueue;
>+import java.util.concurrent.BlockingQueue;
>+import java.util.concurrent.ExecutorService;
>+import java.util.concurrent.Executors;
>+
>+public class ThreadPoolBuilder {
>+	
>+	private ExecutorService threadPool;
>+	
>+	private BlockingQueue<Node<?>> queue;
>+
>+	public ThreadPoolBuilder() {
>+		threadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
>+		queue = new ArrayBlockingQueue<Node<?>>(100);
>+	}
>+	
>+	public <T> void run(DirectedGraphs<T> directedGraphs, IBuildExecutor<T> runnable) throws InterruptedException {
>+		queue.addAll(directedGraphs.getRootNodes());
>+			while (!threadPool.isTerminated()) {
>+				if(directedGraphs.isEmtpy()) {
>+					threadPool.shutdown();
>+				} else {
>+					Node<?>nodeToCompile = queue.take();
>+					directedGraphs.removeNode((Node<T>) nodeToCompile);
>+					threadPool.execute(new BuildRunnable(nodeToCompile, queue, runnable));
>+				}
>+		}
>+	}
>+}

Actions: View | Diff

Attachments on bug 126121: 194413 | 199030 | 233219