in cli/cli.ts [993:1290]
function uploadCoreAsync(opts: UploadOptions) {
let targetConfig = readLocalPxTarget();
let defaultLocale = targetConfig.appTheme.defaultLocale;
let hexCache = path.join("built", "hexcache");
let hexFiles: string[] = [];
if (fs.existsSync(hexCache)) {
hexFiles = fs.readdirSync(hexCache)
.filter(f => /\.hex$/.test(f))
.filter(f => fs.readFileSync(path.join(hexCache, f), { encoding: "utf8" }) != "SKIP")
.map((f) => `@cdnUrl@/compile/${f}`);
pxt.log(`hex cache:\n\t${hexFiles.join('\n\t')}`)
}
let logos = (targetConfig.appTheme as any as Map<string>);
let targetImages = Object.keys(logos)
.filter(k => /(logo|hero)$/i.test(k) && /^\.\//.test(logos[k]));
let targetImagesHashed = pxt.Util.unique(targetImages.map(k => uploadArtFile(logos[k])), url => url);
let targetEditorJs = "";
if (pxt.appTarget.appTheme && pxt.appTarget.appTheme.extendEditor)
targetEditorJs = "@commitCdnUrl@editor.js";
let targetFieldEditorsJs = "";
if (pxt.appTarget.appTheme && pxt.appTarget.appTheme.extendFieldEditors)
targetFieldEditorsJs = "@commitCdnUrl@fieldeditors.js";
let replacements: Map<string> = {
"/sim/simulator.html": "@simUrl@",
"/sim/siminstructions.html": "@partsUrl@",
"/sim/sim.webmanifest": "@relprefix@webmanifest",
"/embed.js": "@targetUrl@@relprefix@embed",
"/cdn/": "@commitCdnUrl@",
"/doccdn/": "@commitCdnUrl@",
"/sim/": "@commitCdnUrl@",
"/blb/": "@blobCdnUrl@",
"/trgblb/": "@targetBlobUrl@",
"@timestamp@": "",
"data-manifest=\"\"": "@manifest@",
"var pxtConfig = null": "var pxtConfig = @cfg@",
"var pxtConfig=null": "var pxtConfig = @cfg@",
"@defaultLocaleStrings@": defaultLocale ? "@commitCdnUrl@" + "locales/" + defaultLocale + "/strings.json" : "",
"@cachedHexFiles@": hexFiles.length ? hexFiles.join("\n") : "",
"@cachedHexFilesEncoded@": encodeURLs(hexFiles),
"@targetEditorJs@": targetEditorJs,
"@targetFieldEditorsJs@": targetFieldEditorsJs,
"@targetImages@": targetImagesHashed.length ? targetImagesHashed.join('\n') : '',
"@targetImagesEncoded@": targetImagesHashed.length ? encodeURLs(targetImagesHashed) : ""
}
if (opts.localDir) {
let cfg: pxt.WebConfig = {
"relprefix": opts.localDir,
"verprefix": "",
"workerjs": opts.localDir + "worker.js",
"monacoworkerjs": opts.localDir + "monacoworker.js",
"gifworkerjs": opts.localDir + "gifjs/gif.worker.js",
"serviceworkerjs": opts.localDir + "serviceworker.js",
"typeScriptWorkerJs": opts.localDir + "tsworker.js",
"pxtVersion": pxtVersion(),
"pxtRelId": "localDirRelId",
"pxtCdnUrl": opts.localDir,
"commitCdnUrl": opts.localDir,
"blobCdnUrl": opts.localDir,
"cdnUrl": opts.localDir,
"targetVersion": opts.pkgversion,
"targetRelId": "",
"targetUrl": "",
"targetId": opts.target,
"simUrl": opts.localDir + "simulator.html",
"simserviceworkerUrl": opts.localDir + "simulatorserviceworker.js",
"simworkerconfigUrl": opts.localDir + "workerConfig.js",
"partsUrl": opts.localDir + "siminstructions.html",
"runUrl": opts.localDir + "run.html",
"docsUrl": opts.localDir + "docs.html",
"multiUrl": opts.localDir + "multi.html",
"asseteditorUrl": opts.localDir + "asseteditor.html",
"skillmapUrl": opts.localDir + "skillmap.html",
"isStatic": true,
}
const targetImagePaths = targetImages.map(k =>
`${opts.localDir}${path.join('./docs', logos[k])}`);
replacements = {
"/embed.js": opts.localDir + "embed.js",
"/cdn/": opts.localDir,
"/doccdn/": opts.localDir,
"/sim/": opts.localDir,
"/blb/": opts.localDir,
"@monacoworkerjs@": `${opts.localDir}monacoworker.js`,
"@gifworkerjs@": `${opts.localDir}gifjs/gif.worker.js`,
"@workerjs@": `${opts.localDir}worker.js`,
"@serviceworkerjs@": `${opts.localDir}serviceworker.js`,
"@timestamp@": `# ver ${new Date().toString()}`,
"var pxtConfig = null": "var pxtConfig = " + JSON.stringify(cfg, null, 4),
"@defaultLocaleStrings@": "",
"@cachedHexFiles@": "",
"@cachedHexFilesEncoded@": "",
"@targetEditorJs@": targetEditorJs ? `${opts.localDir}editor.js` : "",
"@targetFieldEditorsJs@": targetFieldEditorsJs ? `${opts.localDir}fieldeditors.js` : "",
"@targetImages@": targetImages.length ? targetImagePaths.join('\n') : '',
"@targetImagesEncoded@": targetImages.length ? encodeURLs(targetImagePaths) : ''
}
if (!opts.noAppCache) {
replacements["data-manifest=\"\""] = `manifest="${opts.localDir}release.manifest"`;
}
}
let replFiles = [
"index.html",
"embed.js",
"run.html",
"docs.html",
"siminstructions.html",
"codeembed.html",
"release.manifest",
"worker.js",
"serviceworker.js",
"simulatorserviceworker.js",
"monacoworker.js",
"simulator.html",
"sim.manifest",
"sim.webmanifest",
"workerConfig.js",
"multi.html",
"asseteditor.html",
"skillmap.html"
]
// expandHtml is manually called on these files before upload
// runs <!-- @include --> substitutions, fills in locale, etc
let expandFiles = [
"index.html",
"skillmap.html"
]
nodeutil.mkdirP("built/uploadrepl")
function encodeURLs(urls: string[]) {
return urls.map(url => encodeURIComponent(url)).join(";")
}
let uplReqs: Map<BlobReq> = {}
let uploadFileAsync = (p: string) => {
let rdf: Promise<Buffer> = null
if (opts.fileContent) {
let s = U.lookup(opts.fileContent, p)
if (s != null)
rdf = Promise.resolve(Buffer.from(s, "utf8"))
}
if (!rdf) {
if (!fs.existsSync(p))
return undefined;
rdf = readFileAsync(p)
}
const uglify = opts.minify ? require("uglify-js") : undefined;
let fileName = uploadFileName(p)
let mime = U.getMime(p)
const minified = opts.minify && mime == "application/javascript" && fileName !== "target.js";
pxt.log(` ${p} -> ${fileName} (${mime})` + (minified ? ' minified' : ""));
let isText = /^(text\/.*|application\/.*(javascript|json))$/.test(mime)
let content = ""
let data: Buffer;
return rdf.then((rdata: Buffer) => {
data = rdata;
if (isText) {
content = data.toString("utf8")
if (expandFiles.indexOf(fileName) >= 0) {
if (!opts.localDir) {
let m = pxt.appTarget.appTheme as Map<string>
for (let k of Object.keys(m)) {
if (/CDN$/.test(k))
m[k.slice(0, k.length - 3)] = m[k]
}
}
content = server.expandHtml(content)
}
if (/^sim/.test(fileName) || /^workerConfig/.test(fileName)) {
// just force blobs for everything in simulator manifest
content = content.replace(/\/(cdn|sim)\//g, "/blb/")
}
if (minified) {
const res = uglify.minify(content);
if (!res.error) {
content = res.code;
}
else {
pxt.log(` Could not minify ${fileName} ${res.error}`)
}
}
if (replFiles.indexOf(fileName) >= 0) {
for (let from of Object.keys(replacements)) {
content = U.replaceAll(content, from, replacements[from])
}
if (opts.localDir) {
data = Buffer.from(content, "utf8")
} else {
// save it for developer inspection
fs.writeFileSync("built/uploadrepl/" + fileName, content)
}
} else if (fileName == "target.json" || fileName == "target.js") {
let isJs = fileName == "target.js"
if (isJs) content = content.slice(targetJsPrefix.length)
let trg: pxt.TargetBundle = JSON.parse(content)
if (opts.localDir) {
for (let e of trg.appTheme.docMenu)
if (e.path[0] == "/") {
e.path = opts.localDir + "docs" + e.path;
}
trg.appTheme.homeUrl = opts.localDir
// patch icons in bundled packages
Object.keys(trg.bundledpkgs).forEach(pkgid => {
const res = trg.bundledpkgs[pkgid];
// path config before storing
const config = JSON.parse(res[pxt.CONFIG_NAME]) as pxt.PackageConfig;
if (/^\//.test(config.icon)) config.icon = opts.localDir + "docs" + config.icon;
res[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(config);
})
data = Buffer.from((isJs ? targetJsPrefix : '') + nodeutil.stringify(trg), "utf8")
} else {
if (trg.simulator
&& trg.simulator.boardDefinition
&& trg.simulator.boardDefinition.visual) {
let boardDef = trg.simulator.boardDefinition.visual as pxsim.BoardImageDefinition;
if (boardDef.image) {
boardDef.image = uploadArtFile(boardDef.image);
if (boardDef.outlineImage) boardDef.outlineImage = uploadArtFile(boardDef.outlineImage);
}
}
// patch icons in bundled packages
Object.keys(trg.bundledpkgs).forEach(pkgid => {
const res = trg.bundledpkgs[pkgid];
// path config before storing
const config = JSON.parse(res[pxt.CONFIG_NAME]) as pxt.PackageConfig;
if (config.icon) config.icon = uploadArtFile(config.icon);
res[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(config);
})
content = nodeutil.stringify(trg);
if (isJs)
content = targetJsPrefix + content
}
}
} else {
content = data.toString("base64")
}
return Promise.resolve()
}).then(() => {
if (opts.localDir) {
U.assert(!!opts.builtPackaged);
let fn = path.join(opts.builtPackaged, opts.localDir, fileName)
nodeutil.mkdirP(path.dirname(fn))
return minified ? writeFileAsync(fn, content) : writeFileAsync(fn, data)
}
let req = {
encoding: isText ? "utf8" : "base64",
content,
hash: "",
filename: fileName,
size: 0
}
let buf = Buffer.from(req.content, req.encoding)
req.size = buf.length
req.hash = gitHash(buf)
uplReqs[fileName] = req
return Promise.resolve()
})
}
// only keep the last version of each uploadFileName()
opts.fileList = U.values(U.toDictionary(opts.fileList, uploadFileName))
// check size
const maxSize = checkFileSize(opts.fileList);
if (maxSize > 30000000) // 30Mb max
U.userError(`file too big for upload`);
pxt.log('');
if (opts.localDir)
return U.promisePoolAsync(15, opts.fileList, uploadFileAsync)
.then(() => {
pxt.log("Release files written to " + path.join(opts.builtPackaged, opts.localDir))
})
return U.promisePoolAsync(15, opts.fileList, uploadFileAsync)
.then(() =>
opts.githubOnly
? uploadToGitRepoAsync(opts, uplReqs)
: gitUploadAsync(opts, uplReqs))
}