diff --git a/.gitignore b/.gitignore index 6efbcd1..483cdf5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ javacore.* heapdump.* core.* Snap.* - +target +tests/*/*.jar diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildDescription.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildDescription.java new file mode 100755 index 0000000..d057db8 --- /dev/null +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildDescription.java @@ -0,0 +1,42 @@ +/******************************************************************************* + * Copyright (c) 2011 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Jens Kuebler - [126121] - Initial API + * Morgan Stanley - [126121] Updated parallel build patch by fixing concurrency issues + *******************************************************************************/ +package org.eclipse.core.internal.events; + +import org.eclipse.core.resources.IBuildConfiguration; +import org.eclipse.core.resources.IBuildContext; + +import org.eclipse.core.resources.IBuildDescription; + +/** + * + */ +public class BuildDescription implements IBuildDescription { + + private IBuildContext buildContext; + + private IBuildConfiguration buildConfiguration; + + public BuildDescription(IBuildContext buildContext, IBuildConfiguration buildConfiguration) { + super(); + this.buildContext = buildContext; + this.buildConfiguration = buildConfiguration; + } + + public IBuildContext getBuildContext() { + return buildContext; + } + + public IBuildConfiguration getBuildConfiguration() { + return buildConfiguration; + } + +} diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildManager.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildManager.java index fadcdac..605dd44 100644 --- a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildManager.java +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildManager.java @@ -11,6 +11,8 @@ * Anton Leherbauer (Wind River) - [305858] Allow Builder to return null rule * James Blackburn (Broadcom) - [306822] Provide Context for Builder getRule() * Broadcom Corporation - ongoing development + * Jens Kuebler - [126121] - Initial changes to support parallel build + * Morgan Stanley - [126121] Fixing parallel build concurrency issues *******************************************************************************/ package org.eclipse.core.internal.events; @@ -99,10 +101,10 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene private final Set builtProjects = new HashSet(); //the following four fields only apply for the lifetime of a single builder invocation. - protected InternalBuilder currentBuilder; - private DeltaDataTree currentDelta; - private ElementTree currentLastBuiltTree; - private ElementTree currentTree; + protected final ThreadLocal currentBuilder = new ThreadLocal(); + private final ThreadLocal currentDelta = new ThreadLocal(); + private final ThreadLocal currentLastBuiltTree = new ThreadLocal(); + private final ThreadLocal currentTree = new ThreadLocal(); /** * Caches the IResourceDelta for a pair of trees @@ -134,18 +136,29 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene private void basicBuild(int trigger, IncrementalProjectBuilder builder, Map args, MultiStatus status, IProgressMonitor monitor) { try { - currentBuilder = builder; + // flag required for different locking behaviour + boolean notParallel = !(trigger == IncrementalProjectBuilder.PARALLEL_CLEAN_BUILD || trigger == IncrementalProjectBuilder.PARALLEL_FULL_BUILD); + + // Builder impl may not know about Parallel build types and it is not even interesting on the per project level + if (trigger == IncrementalProjectBuilder.PARALLEL_FULL_BUILD) { + trigger = IncrementalProjectBuilder.FULL_BUILD; + } + if (trigger == IncrementalProjectBuilder.PARALLEL_CLEAN_BUILD) { + trigger = IncrementalProjectBuilder.CLEAN_BUILD; + } + + currentBuilder.set(builder); //clear any old requests to forget built state - currentBuilder.clearLastBuiltStateRequests(); + currentBuilder.get().clearLastBuiltStateRequests(); // Figure out want kind of build is needed - boolean clean = trigger == IncrementalProjectBuilder.CLEAN_BUILD; - currentLastBuiltTree = currentBuilder.getLastBuiltTree(); + boolean clean = (trigger == IncrementalProjectBuilder.CLEAN_BUILD); + currentLastBuiltTree.set(currentBuilder.get().getLastBuiltTree()); // Does the build command respond to this trigger? boolean isBuilding = builder.getCommand().isBuilding(trigger); // If no tree is available we have to do a full build - if (!clean && currentLastBuiltTree == null) { + if (!clean && currentLastBuiltTree.get() == null) { // Bug 306746 - Don't promote build to FULL_BUILD if builder doesn't AUTO_BUILD if (trigger == IncrementalProjectBuilder.AUTO_BUILD && !isBuilding) return; @@ -157,24 +170,24 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene //don't build if this builder doesn't respond to the trigger if (!isBuilding) { if (clean) - currentBuilder.setLastBuiltTree(null); + currentBuilder.get().setLastBuiltTree(null); return; } // For incremental builds, grab a pointer to the current state before computing the delta - currentTree = ((trigger == IncrementalProjectBuilder.FULL_BUILD) || clean) ? null : workspace.getElementTree(); + currentTree.set(((trigger == IncrementalProjectBuilder.FULL_BUILD) || clean) ? null : workspace.getElementTree()); int depth = -1; ISchedulingRule rule = null; try { //short-circuit if none of the projects this builder cares about have changed. - if (!needsBuild(currentBuilder, trigger)) { + if (!needsBuild(currentBuilder.get(), trigger)) { //use up the progress allocated for this builder monitor.beginTask("", 1); //$NON-NLS-1$ monitor.done(); return; } rule = builder.getRule(trigger, args); - String name = currentBuilder.getLabel(); + String name = currentBuilder.get().getLabel(); String message; if (name != null) message = NLS.bind(Messages.events_invoking_2, name, builder.getProject().getFullPath()); @@ -183,48 +196,50 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene monitor.subTask(message); hookStartBuild(builder, trigger); // Make the current tree immutable before releasing the WS lock - if (rule != null && currentTree != null) + if (rule != null && currentTree.get() != null) workspace.newWorkingTree(); //release workspace lock while calling builders - depth = getWorkManager().beginUnprotected(); + if (notParallel) { + depth = getWorkManager().beginUnprotected(); + } // Acquire the rule required for running this builder if (rule != null) { Job.getJobManager().beginRule(rule, monitor); // Now that we've acquired the rule, changes may have been made concurrently, ensure we're pointing at the // correct currentTree so delta contains concurrent changes made in areas guarded by the scheduling rule - if (currentTree != null) - currentTree = workspace.getElementTree(); + if (currentTree.get() != null) + currentTree.set(workspace.getElementTree()); } //do the build - SafeRunner.run(getSafeRunnable(trigger, args, status, monitor)); + SafeRunner.run(getSafeRunnable(trigger, args, status, monitor, currentBuilder.get())); } finally { // Re-acquire the WS lock, then release the scheduling rule - if (depth >= 0) + if (notParallel && depth >= 0) getWorkManager().endUnprotected(depth); if (rule != null) Job.getJobManager().endRule(rule); // Be sure to clean up after ourselves. - if (clean || currentBuilder.wasForgetStateRequested()) { - currentBuilder.setLastBuiltTree(null); - } else if (currentBuilder.wasRememberStateRequested()) { + if (clean || currentBuilder.get().wasForgetStateRequested()) { + currentBuilder.get().setLastBuiltTree(null); + } else if (currentBuilder.get().wasRememberStateRequested()) { // If remember last build state, and FULL_BUILD // last tree must be set to => null for next build if (trigger == IncrementalProjectBuilder.FULL_BUILD) - currentBuilder.setLastBuiltTree(null); + currentBuilder.get().setLastBuiltTree(null); // else don't modify the last built tree } else { // remember the current state as the last built state. ElementTree lastTree = workspace.getElementTree(); - lastTree.immutable(); - currentBuilder.setLastBuiltTree(lastTree); +// lastTree.immutable(); + currentBuilder.get().setLastBuiltTree(lastTree); } hookEndBuild(builder); } } finally { - currentBuilder = null; - currentTree = null; - currentLastBuiltTree = null; - currentDelta = null; + currentBuilder.set(null); + currentTree.set(null); + currentLastBuiltTree.set(null); + currentDelta.set(null); } } @@ -234,7 +249,12 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene checkCanceled(trigger, monitor); BuildCommand command = (BuildCommand) commands[i]; IProgressMonitor sub = Policy.subMonitorFor(monitor, 1); - IncrementalProjectBuilder builder = getBuilder(buildConfiguration, command, i, status, context); + IncrementalProjectBuilder builder; + if(IncrementalProjectBuilder.PARALLEL_FULL_BUILD == trigger || IncrementalProjectBuilder.PARALLEL_CLEAN_BUILD == trigger) { + builder = getParallelBuilder(buildConfiguration, command, i, status, context); + } else { + builder = getBuilder(buildConfiguration, command, i, status, context); + } if (builder != null) basicBuild(trigger, builder, command.getArguments(false), status, sub); } @@ -362,26 +382,57 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene * they are given. * @return A status indicating if the build succeeded or failed */ - public IStatus build(IBuildConfiguration[] configs, IBuildConfiguration[] requestedConfigs, int trigger, IProgressMonitor monitor) { - monitor = Policy.monitorFor(monitor); - try { - monitor.beginTask(Messages.events_building_0, TOTAL_BUILD_WORK); - if (!canRun(trigger)) - return Status.OK_STATUS; - try { - hookStartBuild(configs, trigger); - MultiStatus status = new MultiStatus(ResourcesPlugin.PI_RESOURCES, IResourceStatus.BUILD_FAILED, Messages.events_errors, null); - basicBuildLoop(configs, requestedConfigs, trigger, status, monitor); - return status; - } finally { - hookEndBuild(trigger); - } - } finally { - monitor.done(); - if (trigger == IncrementalProjectBuilder.INCREMENTAL_BUILD || trigger == IncrementalProjectBuilder.FULL_BUILD) - autoBuildJob.avoidBuild(); - } - } + public IStatus build(final IBuildConfiguration[] configs, final IBuildConfiguration[] requestedConfigs, final int trigger, final IProgressMonitor monitor) { + if (trigger == IncrementalProjectBuilder.PARALLEL_FULL_BUILD || trigger == IncrementalProjectBuilder.PARALLEL_CLEAN_BUILD) { + + final String buildType = (trigger == IncrementalProjectBuilder.PARALLEL_FULL_BUILD) ? "Parallel full build" : "Parallel clean build"; + Job job = new Job(buildType) { + + @Override + protected IStatus run(IProgressMonitor monitor) { + try { + IBuildConfiguration[] buildConfigs = requestedConfigs.length > 0 ? requestedConfigs : configs; + if (trigger == IncrementalProjectBuilder.PARALLEL_CLEAN_BUILD) { + buildParallel(IncrementalProjectBuilder.PARALLEL_CLEAN_BUILD, monitor, buildConfigs); + } + buildParallel(IncrementalProjectBuilder.PARALLEL_FULL_BUILD, monitor, buildConfigs); + } catch (CoreException e) { + e.printStackTrace(); + return new Status(IStatus.ERROR, ResourcesPlugin.PI_RESOURCES, buildType + " finished with error", e); + } + + return new Status(IStatus.OK, ResourcesPlugin.PI_RESOURCES, buildType + " finished successfully"); + } + }; + + job.setPriority(Job.LONG); + job.schedule(); + + autoBuildJob.avoidBuild(); + + return Status.OK_STATUS; + + } else { + IProgressMonitor myMonitor = Policy.monitorFor(monitor); + try { + myMonitor.beginTask(Messages.events_building_0, TOTAL_BUILD_WORK); + if (!canRun(trigger)) + return Status.OK_STATUS; + try { + hookStartBuild(configs, trigger); + MultiStatus status = new MultiStatus(ResourcesPlugin.PI_RESOURCES, IResourceStatus.BUILD_FAILED, Messages.events_errors, null); + basicBuildLoop(configs, requestedConfigs, trigger, status, myMonitor); + return status; + } finally { + hookEndBuild(trigger); + } + } finally { + myMonitor.done(); + if (trigger == IncrementalProjectBuilder.INCREMENTAL_BUILD || trigger == IncrementalProjectBuilder.FULL_BUILD) + autoBuildJob.avoidBuild(); + } + } + } /** * Runs the builder with the given name on the given project config. @@ -480,13 +531,13 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene } private String debugBuilder() { - return currentBuilder == null ? "" : currentBuilder.getClass().getName(); //$NON-NLS-1$ + return currentBuilder.get() == null ? "" : currentBuilder.get().getClass().getName(); //$NON-NLS-1$ } private String debugProject() { - if (currentBuilder == null) + if (currentBuilder.get() == null) return ""; //$NON-NLS-1$ - return currentBuilder.getProject().getFullPath().toString(); + return currentBuilder.get().getProject().getFullPath().toString(); } /** @@ -498,8 +549,12 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene switch (trigger) { case IncrementalProjectBuilder.FULL_BUILD : return "FULL_BUILD"; //$NON-NLS-1$ + case IncrementalProjectBuilder.PARALLEL_FULL_BUILD : + return "PARALLEL_FULL_BUILD"; //$NON-NLS-1$ case IncrementalProjectBuilder.CLEAN_BUILD : return "CLEAN_BUILD"; //$NON-NLS-1$ + case IncrementalProjectBuilder.PARALLEL_CLEAN_BUILD : + return "PARALLEL_CLEAN_BUILD"; //$NON-NLS-1$ case IncrementalProjectBuilder.AUTO_BUILD : case IncrementalProjectBuilder.INCREMENTAL_BUILD : default : @@ -581,18 +636,20 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene //try to match on builder index, but if not match is found, use the builder name and config name //this is because older workspace versions did not store builder infos in build spec order BuilderPersistentInfo nameMatch = null; - for (Iterator it = infos.iterator(); it.hasNext();) { - BuilderPersistentInfo info = it.next(); - // match on name, config name and build spec index if known - // Note: the config name may be null for builders that don't support configurations, or old workspaces - if (info.getBuilderName().equals(builderName) && (info.getConfigName() == null || info.getConfigName().equals(configName))) { - //we have found a match on name alone - if (nameMatch == null) - nameMatch = info; - //see if the index matches - if (buildSpecIndex == -1 || info.getBuildSpecIndex() == -1 || buildSpecIndex == info.getBuildSpecIndex()) - return info; - } + synchronized(infos) { + for (Iterator it = infos.iterator(); it.hasNext();) { + BuilderPersistentInfo info = it.next(); + // match on name, config name and build spec index if known + // Note: the config name may be null for builders that don't support configurations, or old workspaces + if (info.getBuilderName().equals(builderName) && (info.getConfigName() == null || info.getConfigName().equals(configName))) { + //we have found a match on name alone + if (nameMatch == null) + nameMatch = info; + //see if the index matches + if (buildSpecIndex == -1 || info.getBuildSpecIndex() == -1 || buildSpecIndex == info.getBuildSpecIndex()) + return info; + } + } } //no exact index match, so return name match, if any return nameMatch; @@ -645,7 +702,7 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene IResourceDelta getDelta(IProject project) { try { lock.acquire(); - if (currentTree == null) { + if (currentTree.get() == null) { if (Policy.DEBUG_BUILD_FAILURE) Policy.debug("Build: no tree for delta " + debugBuilder() + " [" + debugProject() + "]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ return null; @@ -657,7 +714,7 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene return null; } //check if this project has changed - if (currentDelta != null && currentDelta.findNodeAt(project.getFullPath()) == null) { + if (currentDelta.get() != null && currentDelta.get().findNodeAt(project.getFullPath()) == null) { //if the project never existed (not in delta and not in current tree), return null if (!project.exists()) return null; @@ -665,7 +722,7 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene return ResourceDeltaFactory.newEmptyDelta(project); } //now check against the cache - IResourceDelta result = (IResourceDelta) deltaCache.getDelta(project.getFullPath(), currentLastBuiltTree, currentTree); + IResourceDelta result = (IResourceDelta) deltaCache.getDelta(project.getFullPath(), currentLastBuiltTree.get(), currentTree.get()); if (result != null) return result; @@ -674,8 +731,8 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene startTime = System.currentTimeMillis(); Policy.debug("Computing delta for project: " + project.getName()); //$NON-NLS-1$ } - result = ResourceDeltaFactory.computeDelta(workspace, currentLastBuiltTree, currentTree, project.getFullPath(), -1); - deltaCache.cache(project.getFullPath(), currentLastBuiltTree, currentTree, result); + result = ResourceDeltaFactory.computeDelta(workspace, currentLastBuiltTree.get(), currentTree.get(), project.getFullPath(), -1); + deltaCache.cache(project.getFullPath(), currentLastBuiltTree.get(), currentTree.get(), result); if (Policy.DEBUG_BUILD_FAILURE && result == null) Policy.debug("Build: no delta " + debugBuilder() + " [" + debugProject() + "] " + project.getFullPath()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ if (Policy.DEBUG_BUILD_DELTA) { @@ -692,7 +749,7 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene /** * Returns the safe runnable instance for invoking a builder */ - private ISafeRunnable getSafeRunnable(final int trigger, final Map args, final MultiStatus status, final IProgressMonitor monitor) { + private ISafeRunnable getSafeRunnable(final int trigger, final Map args, final MultiStatus status, final IProgressMonitor monitor, final InternalBuilder builder) { return new ISafeRunnable() { public void handleException(Throwable e) { if (e instanceof OperationCanceledException) { @@ -700,18 +757,18 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene Policy.debug("Build canceled"); //$NON-NLS-1$ //just discard built state when a builder cancels, to ensure //that it is called again on the very next build. - currentBuilder.forgetLastBuiltState(); + builder.forgetLastBuiltState(); throw (OperationCanceledException) e; } //ResourceStats.buildException(e); // don't log the exception....it is already being logged in SafeRunner#run //add a generic message to the MultiStatus - String builderName = currentBuilder.getLabel(); + String builderName = builder.getLabel(); if (builderName == null || builderName.length() == 0) - builderName = currentBuilder.getClass().getName(); - String pluginId = currentBuilder.getPluginId(); - String message = NLS.bind(Messages.events_builderError, builderName, currentBuilder.getProject().getName()); + builderName = builder.getClass().getName(); + String pluginId = builder.getPluginId(); + String message = NLS.bind(Messages.events_builderError, builderName, builder.getProject().getName()); status.add(new Status(IStatus.ERROR, pluginId, IResourceStatus.BUILD_FAILED, message, e)); //add the exception status to the MultiStatus @@ -722,13 +779,15 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene public void run() throws Exception { IProject[] prereqs = null; //invoke the appropriate build method depending on the trigger - if (trigger != IncrementalProjectBuilder.CLEAN_BUILD) - prereqs = currentBuilder.build(trigger, args, monitor); - else - currentBuilder.clean(monitor); - if (prereqs == null) - prereqs = new IProject[0]; - currentBuilder.setInterestingProjects(prereqs.clone()); + if (trigger == IncrementalProjectBuilder.CLEAN_BUILD) { + builder.clean(monitor); + } else { + prereqs = builder.build(trigger, args, monitor); + } + + if (prereqs == null) prereqs = new IProject[0]; + + builder.setInterestingProjects(prereqs.clone()); } }; } @@ -920,9 +979,9 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene * to the given project, and false otherwise. */ private boolean isInterestingProject(IProject project) { - if (project.equals(currentBuilder.getProject())) + if (project.equals(currentBuilder.get().getProject())) return true; - IProject[] interestingProjects = currentBuilder.getInterestingProjects(); + IProject[] interestingProjects = currentBuilder.get().getInterestingProjects(); for (int i = 0; i < interestingProjects.length; i++) { if (interestingProjects[i].equals(project)) { return true; @@ -950,7 +1009,7 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene case IncrementalProjectBuilder.FULL_BUILD : return true; case IncrementalProjectBuilder.INCREMENTAL_BUILD : - if (currentBuilder.callOnEmptyDelta()) + if (currentBuilder.get().callOnEmptyDelta()) return true; //fall through and check if there is a delta } @@ -959,20 +1018,20 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene ElementTree oldTree = builder.getLastBuiltTree(); ElementTree newTree = workspace.getElementTree(); long start = System.currentTimeMillis(); - currentDelta = (DeltaDataTree) deltaTreeCache.getDelta(null, oldTree, newTree); - if (currentDelta == null) { + currentDelta.set((DeltaDataTree) deltaTreeCache.getDelta(null, oldTree, newTree)); + if (currentDelta.get() == null) { if (Policy.DEBUG_BUILD_NEEDED) { String message = "Checking if need to build. Starting delta computation between: " + oldTree.toString() + " and " + newTree.toString(); //$NON-NLS-1$ //$NON-NLS-2$ Policy.debug(message); } - currentDelta = newTree.getDataTree().forwardDeltaWith(oldTree.getDataTree(), ResourceComparator.getBuildComparator()); + currentDelta.set(newTree.getDataTree().forwardDeltaWith(oldTree.getDataTree(), ResourceComparator.getBuildComparator())); if (Policy.DEBUG_BUILD_NEEDED) Policy.debug("End delta computation. (" + (System.currentTimeMillis() - start) + "ms)."); //$NON-NLS-1$ //$NON-NLS-2$ - deltaTreeCache.cache(null, oldTree, newTree, currentDelta); + deltaTreeCache.cache(null, oldTree, newTree, currentDelta.get()); } //search for the builder's project - if (currentDelta.findNodeAt(builder.getProject().getFullPath()) != null) { + if (currentDelta.get().findNodeAt(builder.getProject().getFullPath()) != null) { if (Policy.DEBUG_BUILD_NEEDED) Policy.debug(toString(builder) + " needs building because of changes in: " + builder.getProject().getName()); //$NON-NLS-1$ return true; @@ -981,7 +1040,7 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene //search for builder's interesting projects IProject[] projects = builder.getInterestingProjects(); for (int i = 0; i < projects.length; i++) { - if (currentDelta.findNodeAt(projects[i].getFullPath()) != null) { + if (currentDelta.get().findNodeAt(projects[i].getFullPath()) != null) { if (Policy.DEBUG_BUILD_NEEDED) Policy.debug(toString(builder) + " needs building because of changes in: " + projects[i].getName()); //$NON-NLS-1$ return true; @@ -1143,4 +1202,98 @@ public class BuildManager implements ICoreConstants, IManager, ILifecycleListene Policy.log(status); return workspace.getRoot(); } + + public void buildParallel(final int trigger, IProgressMonitor monitor, IBuildConfiguration[] configs) throws CoreException { + + final boolean clean = (trigger == IncrementalProjectBuilder.PARALLEL_CLEAN_BUILD); + + long start = System.currentTimeMillis(); + if (clean) { + Policy.log(new ResourceStatus(IStatus.INFO, "Starting parallel cleaning of " + configs.length + " projects...")); + } else { + Policy.log(new ResourceStatus(IStatus.INFO, "Starting parallel build of " + configs.length + " projects...")); + } + final SharedProgressMonitor sharedProgress = new SharedProgressMonitor(Policy.monitorFor(monitor)); + + List projects = new ArrayList(); + DirectedGraphs directedGraphs = new DirectedGraphs(); + for (IBuildConfiguration config : configs) { + projects.add(config.getProject()); + directedGraphs.addNode(new Node(config.getProject())); + } + + // cleaning does not require building the graph + if (trigger == IncrementalProjectBuilder.PARALLEL_FULL_BUILD) { + for (IProject project : projects) { + if (!project.isAccessible()) + continue; + ProjectDescription desc = ((Project)project).internalGetDescription(); + if (desc == null) + continue; + IProject[] refs = desc.getAllReferences(false); + Node node = directedGraphs.getNodeWithContent(project); + for (IProject referencedProject : refs) { + Node node2 = directedGraphs.getNodeWithContent(referencedProject); + if(node2 != null) { + node2.addOutgoingEdge(node); + } + } + } + } + + hookStartBuild(configs, trigger); + + final MultiStatus status = new MultiStatus(ResourcesPlugin.PI_RESOURCES, IResourceStatus.INTERNAL_ERROR, Messages.events_errors, null); + + if (clean) { + sharedProgress.beginTask("Cleaning projects in parallel:", projects.size()); + } else { + sharedProgress.beginTask("Building projects in parallel:", projects.size()); + } + + ThreadPoolBuilder threadPoolBuilder = new ThreadPoolBuilder(); + try { + IBuildExecutor buildExecutor = new IBuildExecutor() { + public void executeBuild(IBuildDescription buildDescription, SharedProgressMonitor sharedProgress) { + IProgressMonitor progressMonitor = new NullProgressMonitor(); + try { + sharedProgress.addProjectName(buildDescription.getBuildConfiguration().getProject().getName()); + basicBuild(buildDescription.getBuildConfiguration(), trigger, buildDescription.getBuildContext(), status, progressMonitor); + } finally { + sharedProgress.removeProjectName(buildDescription.getBuildConfiguration().getProject().getName()); + sharedProgress.worked(1); + } + }}; + threadPoolBuilder.run(directedGraphs, buildExecutor, sharedProgress); + } catch (InterruptedException e) { + status.add(new Status(IStatus.ERROR, ResourcesPlugin.PI_RESOURCES, Messages.events_errors, e)); + } finally { + hookEndBuild(trigger); + long finish = System.currentTimeMillis(); + if (clean) { + Policy.log(new ResourceStatus(IStatus.INFO, "Finished parallel clean in " + (finish - start) + "ms")); + } else { + Policy.log(new ResourceStatus(IStatus.INFO, "Finished parallel build in " + (finish - start) + "ms")); + } + sharedProgress.done(); + } + } + + private IncrementalProjectBuilder getParallelBuilder(IBuildConfiguration buildConfiguration, BuildCommand command, int buildSpecIndex, MultiStatus status, IBuildContext buildContext) throws CoreException { + // always construct a new builder for parallel builds (think about pooling here) + InternalBuilder result = initializeBuilder(command.getBuilderName(), buildConfiguration, buildSpecIndex, status); + command.setBuilders(result); + result.setCommand(command); + result.setBuildConfig(buildConfiguration); + result.startupOnInitialize(); + // } + + if (!validateNature(result, command.getBuilderName())) { + //skip this builder and null its last built tree because it is invalid + //if the nature gets added or re-enabled a full build will be triggered + result.setLastBuiltTree(null); + return null; + } + return (IncrementalProjectBuilder) result; + } } diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildRunnable.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildRunnable.java new file mode 100755 index 0000000..995a745 --- /dev/null +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/BuildRunnable.java @@ -0,0 +1,52 @@ +/******************************************************************************* + * Copyright (c) 2000, 2005 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Jens Kuebler - [126121] - Initial changes to support parallel build + * Morgan Stanley - [126121] Fixing parallel build concurrency issues +*******************************************************************************/ +package org.eclipse.core.internal.events; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; + +import org.eclipse.core.internal.resources.BuildConfiguration; +import org.eclipse.core.resources.IProject; + +// The worker instance that is executed in a ThreadPool worker thread +public final class BuildRunnable implements Runnable { + + private final Node currentNode; + private final BlockingQueue> queue; + private final IBuildExecutor executor; + private final SharedProgressMonitor sharedProgress; + + protected BuildRunnable(Node currentNode, BlockingQueue> queue, IBuildExecutor executor, SharedProgressMonitor sharedMonitor) { + this.currentNode = currentNode; + this.queue = queue; + this.executor = executor; + this.sharedProgress = sharedMonitor; + } + + public void run() { + if (sharedProgress.isCanceled()) { + return; + } +// System.out.println("start compile " + currentNode.getContent()); //$NON-NLS-1$ + BuildDescription buildDescription = new BuildDescription(null, new BuildConfiguration(currentNode.getContent())); + executor.executeBuild(buildDescription, sharedProgress); +// System.out.println("end compile " + currentNode.getContent()); //$NON-NLS-1$ + List> outgoingNodes = new ArrayList>(currentNode.getOutgoingEdges()); + currentNode.removeAllOutgoingEdges(); + for (Node node : outgoingNodes) { + if(node.getIncomingEdges().isEmpty()) { + queue.add(node); + } + } + } +} diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/DirectedGraphs.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/DirectedGraphs.java new file mode 100755 index 0000000..116fdf5 --- /dev/null +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/DirectedGraphs.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2000, 2005 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Jens Kuebler - [126121] - Initial changes to support parallel build + *******************************************************************************/ +package org.eclipse.core.internal.events; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * A directed graph holds all nodes. Nodes need not be connected which is why a container object is required + * + * @author jens + * + */ +public class DirectedGraphs { + + private Set> nodes; + + public DirectedGraphs() { + nodes = Collections.synchronizedSet(new HashSet>()); + } + + public void addNode(Node node) { + nodes.add(node); + } + + public void removeNode(Node node) { + nodes.remove(node); + } + + public void addAll(Collection> nodesToBeAdded) { + nodes.addAll(nodesToBeAdded); + } + + public Node getNodeWithContent(T content) { + for (Node node : nodes) { + if(content.equals(node.getContent())) { + return node; + } + } + return null; + } + + public synchronized Set> getRootNodes() { + Set> set = new LinkedHashSet>(); + for (Node node : nodes) { + if(node.getIncomingEdges().isEmpty()) { + set.add(node); + } + } + return set; + } + + public boolean isEmtpy() { + return nodes.isEmpty(); + } +} diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/IBuildExecutor.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/IBuildExecutor.java new file mode 100755 index 0000000..9364bf7 --- /dev/null +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/IBuildExecutor.java @@ -0,0 +1,23 @@ +/******************************************************************************* + * Copyright (c) 2011 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Jens Kuebler - [126121] - Initial changes to support parallel build + *******************************************************************************/ +package org.eclipse.core.internal.events; + +import org.eclipse.core.resources.IBuildDescription; + + +/** + * + */ +public interface IBuildExecutor { + + public void executeBuild(IBuildDescription buildDescription, SharedProgressMonitor sharedProgress); + +} diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/Node.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/Node.java new file mode 100755 index 0000000..3d3f454 --- /dev/null +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/Node.java @@ -0,0 +1,138 @@ +/******************************************************************************* + * Copyright (c) 2011 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Jens Kuebler - [126121] - Initial changes to support parallel build + *******************************************************************************/ +package org.eclipse.core.internal.events; + +import java.text.MessageFormat; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * A simple node object having references to other node objects + * + * @param + */ +public class Node { + + private Set> incomingEdges; + + private Set> outgoingEdges; + + private T content; + + public Node(T content) { + this.content = content; + incomingEdges = new LinkedHashSet>(); + outgoingEdges = new LinkedHashSet>(); + } + + public T getContent() { + return content; + } + + public void setContent(T content) { + this.content = content; + } + + public synchronized Set> getIncomingEdges() { + return incomingEdges; + } + + public synchronized Set> getOutgoingEdges() { + return outgoingEdges; + } + + public synchronized void addIncomingEdge(Node node) { + if(!incomingEdges.contains(node)) { + incomingEdges.add(node); + node.addOutgoingEdge(this); + } + } + + public synchronized void addOutgoingEdge(Node node) { + if(!outgoingEdges.contains(node)) { + outgoingEdges.add(node); + node.addIncomingEdge(this); + } + } + + public synchronized void removeOutgoingEdge(Node node) { + if(outgoingEdges.contains(node)) { + outgoingEdges.remove(node); + node.removeIncomingEdge(this); + } + } + + public synchronized void removeIncomingEdge(Node node) { + if(incomingEdges.contains(node)) { + incomingEdges.remove(node); + node.removeOutgoingEdge(this); + } + } + + public synchronized void removeAllOutgoingEdges() { + for (Iterator> iterator = outgoingEdges.iterator(); iterator.hasNext();) { + Node node = iterator.next(); + node.internalRemoveIncomingEdge(this); + iterator.remove(); + } + } + + public synchronized void removeAllIncomingEdges() { + for (Iterator> iterator = incomingEdges.iterator(); iterator.hasNext();) { + Node node = iterator.next(); + node.internalRemoveOutgoingEdge(this); + iterator.remove(); + } + } + + synchronized void internalRemoveOutgoingEdge(Node node) { + outgoingEdges.remove(node); + } + + synchronized void internalRemoveIncomingEdge(Node node) { + incomingEdges.remove(node); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((content == null) ? 0 : content.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Node other = (Node) obj; + if (content == null) { + if (other.content != null) + return false; + } else if (!content.equals(other.content)) + return false; + return true; + } + + @Override + public String toString() { + return MessageFormat.format("Node [content={0}]", content); //$NON-NLS-1$ + } + + + +} diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/SharedProgressMonitor.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/SharedProgressMonitor.java new file mode 100755 index 0000000..57d6785 --- /dev/null +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/SharedProgressMonitor.java @@ -0,0 +1,89 @@ +/******************************************************************************* + * Copyright (c) 2011 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Morgan Stanley - [126121] Fixing parallel build concurrency issues + *******************************************************************************/ +package org.eclipse.core.internal.events; + +import java.util.LinkedHashSet; +import java.util.Set; + +import org.eclipse.core.runtime.IProgressMonitor; + +/** + * Simple thread-safe {@link IProgressMonitor} wrapper used by the parallel builder to add/remove names of + * projects that are actually building. + */ +public class SharedProgressMonitor implements IProgressMonitor { + + private final IProgressMonitor progressMonitor; + private final Set projectNames = new LinkedHashSet(); + private String taskNamePrefix = ""; + + public SharedProgressMonitor(IProgressMonitor progressMonitor) { + this.progressMonitor = progressMonitor; + } + + public synchronized void beginTask(String name, int totalWork) { + taskNamePrefix = name; + progressMonitor.beginTask(name, totalWork); + } + + public synchronized void done() { + progressMonitor.done(); + } + + public synchronized void internalWorked(double work) { + progressMonitor.internalWorked(work); + } + + public boolean isCanceled() { + return progressMonitor.isCanceled(); + } + + public synchronized void setCanceled(boolean value) { + progressMonitor.setCanceled(value); + } + + public synchronized void setTaskName(String name) { + progressMonitor.setTaskName(name); + } + + public synchronized void subTask(String name) { + progressMonitor.subTask(name); + } + + public synchronized void worked(int work) { + progressMonitor.worked(work); + } + + public synchronized void addProjectName(String projName) { + projectNames.add(projName); + setTaskName(taskNamePrefix + " " + getProjectNames(projectNames)); + } + + public synchronized void removeProjectName(String projName) { + projectNames.remove(projName); + setTaskName(taskNamePrefix + " " + getProjectNames(projectNames)); + } + + private static String getProjectNames(Set projectNames) { + StringBuilder builder = new StringBuilder("["); + boolean first = true; + for (String name : projectNames) { + if (first) { + first = false; + } else { + builder.append(", "); + } + builder.append(name); + } + builder.append("]"); + return builder.toString(); + } +} diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/ThreadPoolBuilder.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/ThreadPoolBuilder.java new file mode 100755 index 0000000..3009caa --- /dev/null +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/internal/events/ThreadPoolBuilder.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright (c) 2011 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Jens Kuebler - [126121] - Initial changes to support parallel build + * Morgan Stanley - [126121] Fixing parallel build concurrency issues + *******************************************************************************/ +package org.eclipse.core.internal.events; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.eclipse.core.resources.IProject; + +public class ThreadPoolBuilder { + + private ExecutorService threadPool; + + private BlockingQueue> queue; + + public ThreadPoolBuilder() { + final int threadNum = Runtime.getRuntime().availableProcessors() > 1 ? Runtime.getRuntime().availableProcessors() - 1 : 1; + threadPool = Executors.newFixedThreadPool(threadNum); + queue = new ArrayBlockingQueue>(6000); + } + + public void run(DirectedGraphs directedGraphs, IBuildExecutor runnable, SharedProgressMonitor sharedProgress) throws InterruptedException { +// System.out.println("ThreadPool started..."); + queue.addAll(directedGraphs.getRootNodes()); + while (!threadPool.isShutdown()) { + if(directedGraphs.isEmtpy() || sharedProgress.isCanceled()) { + threadPool.shutdown(); + } else { + NodenodeToCompile = queue.take(); + directedGraphs.removeNode((Node) nodeToCompile); + threadPool.execute(new BuildRunnable(nodeToCompile, queue, runnable, sharedProgress)); + } + } + // blocking the main builder thread (Worker-N) until all the threads finishes + threadPool.awaitTermination(60, TimeUnit.SECONDS); +// System.out.println("ThreadPool finished..."); + } +} diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/resources/IBuildDescription.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/resources/IBuildDescription.java new file mode 100755 index 0000000..4bc1cdf --- /dev/null +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/resources/IBuildDescription.java @@ -0,0 +1,22 @@ +/******************************************************************************* + * Copyright (c) 2011 IBM Corporation and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * IBM Corporation - initial API and implementation + *******************************************************************************/ +package org.eclipse.core.resources; + +/** + * + */ +public interface IBuildDescription { + + IBuildContext getBuildContext(); + + IBuildConfiguration getBuildConfiguration(); + +} diff --git a/bundles/org.eclipse.core.resources/src/org/eclipse/core/resources/IncrementalProjectBuilder.java b/bundles/org.eclipse.core.resources/src/org/eclipse/core/resources/IncrementalProjectBuilder.java index 4c3a0f2..6e8e354 100644 --- a/bundles/org.eclipse.core.resources/src/org/eclipse/core/resources/IncrementalProjectBuilder.java +++ b/bundles/org.eclipse.core.resources/src/org/eclipse/core/resources/IncrementalProjectBuilder.java @@ -11,6 +11,8 @@ * Anton Leherbauer (Wind River) - [305858] Allow Builder to return null rule * James Blackburn (Broadcom) - [306822] Provide Context for Builder getRule() * Broadcom Corporation - build configurations and references + * Jens Kuebler - [126121] - Initial changes to support parallel build + * Morgan Stanley - [126121] Adding support for parallel clean build *******************************************************************************/ package org.eclipse.core.resources; @@ -50,6 +52,9 @@ public abstract class IncrementalProjectBuilder extends InternalBuilder implemen * @see IWorkspace#build(int, IProgressMonitor) */ public static final int FULL_BUILD = 6; + + public static final int PARALLEL_FULL_BUILD = 7; + /** * Build kind constant (value 9) indicating an automatic build request. When * autobuild is turned on, these builds are triggered automatically whenever @@ -84,6 +89,8 @@ public abstract class IncrementalProjectBuilder extends InternalBuilder implemen * @since 3.0 */ public static final int CLEAN_BUILD = 15; + + public static final int PARALLEL_CLEAN_BUILD = 16; /** * Runs this builder in the specified manner. Subclasses should implement @@ -451,7 +458,7 @@ public abstract class IncrementalProjectBuilder extends InternalBuilder implemen * @since 3.6 */ public ISchedulingRule getRule(int kind, Map args) { - return getRule(); + return getProject(); } /**