| /* This Source Code Form is subject to the terms of the Mozilla Public |
| * License, v. 2.0. If a copy of the MPL was not distributed with this |
| * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ |
| |
| import {clone} from "merge"; |
| import merge from "./merge"; |
| import slugid from "slugid"; |
| import taskcluster from "taskcluster-client"; |
| import * as image_builder from "./image_builder"; |
| |
| let maps = []; |
| let filters = []; |
| |
| let tasks = new Map(); |
| let tags = new Map(); |
| let image_tasks = new Map(); |
| let parameters = {}; |
| |
| let queue = new taskcluster.Queue({ |
| rootUrl: process.env.TASKCLUSTER_PROXY_URL, |
| }); |
| |
| function fromNow(hours) { |
| let d = new Date(); |
| d.setHours(d.getHours() + (hours|0)); |
| return d.toJSON(); |
| } |
| |
| function parseRoutes(routes) { |
| let rv = [ |
| `tc-treeherder.v2.${process.env.TC_PROJECT}.${process.env.NSS_HEAD_REVISION}.${process.env.NSS_PUSHLOG_ID}`, |
| ...routes |
| ]; |
| |
| // Notify about failures (except on try). |
| // Turned off, too noisy. |
| /*if (process.env.TC_PROJECT != "nss-try") { |
| rv.push(`notify.email.${process.env.TC_OWNER}.on-failed`, |
| `notify.email.${process.env.TC_OWNER}.on-exception`); |
| }*/ |
| |
| return rv; |
| } |
| |
| function parseFeatures(list) { |
| return list.reduce((map, feature) => { |
| map[feature] = true; |
| return map; |
| }, {}); |
| } |
| |
| function parseArtifacts(artifacts) { |
| let copy = clone(artifacts); |
| Object.keys(copy).forEach(key => { |
| copy[key].expires = fromNow(copy[key].expires); |
| }); |
| return copy; |
| } |
| |
| function parseCollection(name) { |
| let collection = {}; |
| collection[name] = true; |
| return collection; |
| } |
| |
| function parseTreeherder(def) { |
| let treeherder = { |
| build: { |
| platform: def.platform |
| }, |
| machine: { |
| platform: def.platform |
| }, |
| symbol: def.symbol, |
| jobKind: def.kind |
| }; |
| |
| if (def.group) { |
| treeherder.groupSymbol = def.group; |
| } |
| |
| if (def.collection) { |
| treeherder.collection = parseCollection(def.collection); |
| } |
| |
| if (def.tier) { |
| treeherder.tier = def.tier; |
| } |
| |
| return treeherder; |
| } |
| |
| function convertTask(def) { |
| let scopes = []; |
| let dependencies = []; |
| |
| let env = merge({ |
| NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, |
| NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION, |
| NSS_MAX_MP_PBE_ITERATION_COUNT: "100", |
| }, def.env || {}); |
| |
| if (def.parent) { |
| dependencies.push(def.parent); |
| env.TC_PARENT_TASK_ID = def.parent; |
| } |
| if (def.parents) { |
| dependencies = dependencies.concat(def.parents); |
| } |
| |
| if (def.tests) { |
| env.NSS_TESTS = def.tests; |
| } |
| |
| if (def.cycle) { |
| env.NSS_CYCLES = def.cycle; |
| } |
| if (def.kind === "build") { |
| // Disable leak checking during builds (bug 1579290). |
| if (env.ASAN_OPTIONS) { |
| env.ASAN_OPTIONS += ":detect_leaks=0"; |
| } else { |
| env.ASAN_OPTIONS = "detect_leaks=0"; |
| } |
| } |
| |
| let payload = { |
| env, |
| command: def.command, |
| maxRunTime: def.maxRunTime || 3600 |
| }; |
| |
| if (def.image) { |
| payload.image = def.image; |
| } |
| |
| if (def.artifacts) { |
| payload.artifacts = parseArtifacts(def.artifacts); |
| } |
| |
| if (def.features) { |
| payload.features = parseFeatures(def.features); |
| |
| if (payload.features.allowPtrace) { |
| scopes.push("docker-worker:feature:allowPtrace"); |
| } |
| } |
| |
| if (def.scopes) { |
| // Need to add existing scopes in the task definition |
| scopes.push.apply(scopes, def.scopes) |
| } |
| |
| let extra = Object.assign({ |
| treeherder: parseTreeherder(def) |
| }, parameters); |
| |
| return { |
| provisionerId: def.provisioner || `nss-${process.env.MOZ_SCM_LEVEL}`, |
| workerType: def.workerType || "linux", |
| schedulerId: process.env.TC_SCHEDULER_ID, |
| taskGroupId: process.env.TASK_ID, |
| |
| scopes, |
| created: fromNow(0), |
| deadline: fromNow(24), |
| |
| dependencies, |
| requires: def.requires || "all-completed", |
| routes: parseRoutes(def.routes || []), |
| |
| metadata: { |
| name: def.name, |
| description: def.name, |
| owner: process.env.TC_OWNER, |
| source: process.env.TC_SOURCE |
| }, |
| |
| payload, |
| extra, |
| }; |
| } |
| |
| export function map(fun) { |
| maps.push(fun); |
| } |
| |
| export function filter(fun) { |
| filters.push(fun); |
| } |
| |
| export function addParameters(params) { |
| parameters = Object.assign(parameters, params); |
| } |
| |
| export function clearFilters(fun) { |
| filters = []; |
| } |
| |
| export function taggedTasks(tag) { |
| return tags[tag]; |
| } |
| |
| export function scheduleTask(def) { |
| let taskId = slugid.v4(); |
| tasks.set(taskId, merge({}, def)); |
| return taskId; |
| } |
| |
| export async function submit() { |
| let promises = new Map(); |
| |
| for (let [taskId, task] of tasks) { |
| // Allow filtering tasks before we schedule them. |
| if (!filters.every(filter => filter(task))) { |
| continue; |
| } |
| |
| // Allow changing tasks before we schedule them. |
| maps.forEach(map => { task = map(merge({}, task)) }); |
| |
| let log_id = `${task.name} @ ${task.platform}[${task.collection || "opt"}]`; |
| if (task.group) { |
| log_id = `${task.group}::${log_id}`; |
| } |
| console.log(`+ Submitting ${log_id}.`); |
| |
| // Index that task for each tag specified |
| if(task.tags) { |
| task.tags.map(tag => { |
| if(!tags[tag]) { |
| tags[tag] = []; |
| } |
| tags[tag].push(taskId); |
| }); |
| } |
| |
| let parent = task.parent; |
| |
| // Convert the task definition. |
| task = await convertTask(task); |
| |
| // Convert the docker image definition. |
| let image_def = task.payload.image; |
| if (image_def && image_def.hasOwnProperty("path")) { |
| let key = `${image_def.name}:${image_def.path}`; |
| let data = {}; |
| |
| // Check the cache first. |
| if (image_tasks.has(key)) { |
| data = image_tasks.get(key); |
| } else { |
| data.taskId = await image_builder.findTask(image_def); |
| data.isPending = !data.taskId; |
| |
| // No task found. |
| if (data.isPending) { |
| let image_task = await image_builder.buildTask(image_def); |
| |
| // Schedule a new image builder task immediately. |
| data.taskId = slugid.v4(); |
| |
| try { |
| await queue.createTask(data.taskId, convertTask(image_task)); |
| } catch (e) { |
| console.error("! FAIL: Scheduling image builder task failed."); |
| continue; /* Skip this task on failure. */ |
| } |
| } |
| |
| // Store in cache. |
| image_tasks.set(key, data); |
| } |
| |
| if (data.isPending) { |
| task.dependencies.push(data.taskId); |
| } |
| |
| task.payload.image = { |
| path: "public/image.tar", |
| taskId: data.taskId, |
| type: "task-image" |
| }; |
| } |
| |
| // Wait for the parent task to be created before scheduling dependants. |
| let predecessor = parent ? promises.get(parent) : Promise.resolve(); |
| |
| promises.set(taskId, predecessor.then(() => { |
| // Schedule the task. |
| return queue.createTask(taskId, task).catch(err => { |
| console.error(`! FAIL: Scheduling ${log_id} failed.`, err); |
| }); |
| })); |
| } |
| |
| // Wait for all requests to finish. |
| if (promises.length) { |
| await Promise.all([...promises.values()]); |
| console.log("=== Total:", promises.length, "tasks. ==="); |
| } |
| |
| tasks.clear(); |
| } |