CoCalc’s goal is to provide the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual use to large groups and classes.
CoCalc’s goal is to provide the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual use to large groups and classes.
Path: blob/master/src/workspaces.py
Views: 486
#!/usr/bin/env python31"""2PURPOSE: Automate building, installing, and publishing our modules.3This is like a little clone of "lerna" for our purposes.45NOTE: I wrote this initially using npm and with the goal of publishing6to npmjs.com. Now I don't care at all about publishing to npmjs.com,7and we're using pnpm. So this is being turned into a package just8for cleaning/installing/building.910TEST:11- This should always work: "mypy workspaces.py"12"""131415import argparse, json, os, platform, shutil, subprocess, sys, time1617from typing import Any, Optional, Callable, List1819MAX_PACKAGE_LOCK_SIZE_MB = 5202122def newest_file(path: str) -> str:23if platform.system() != 'Darwin':24# See https://gist.github.com/brwyatt/c21a888d79927cb476a4 for this Linux25# version:26cmd = 'find . -type f -printf "%C@ %p\n" | sort -rn | head -n 1 | cut -d" " -f2'27else:28# but we had to rewrite this as suggested at29# https://unix.stackexchange.com/questions/272491/bash-error-find-printf-unknown-primary-or-operator30# etc to work on MacOS.31cmd = 'find . -type f -print0 | xargs -0r stat -f "%Fc %N" | sort -rn | head -n 1 | cut -d" " -f2'32return os.popen(f'cd "{path}" && {cmd}').read().strip()333435SUCCESSFUL_BUILD = ".successful-build"363738def needs_build(package: str) -> bool:39# Code below was hopelessly naive, e.g, a failed build would not get retried.40# We only need to do a build if the newest file in the tree is not41# in the dist directory.42path = os.path.join(os.path.dirname(__file__), package)43if not os.path.exists(os.path.join(path, 'dist')):44return True45newest = newest_file(path)46return not newest.startswith('./' + SUCCESSFUL_BUILD)474849def handle_path(s: str,50path: Optional[str] = None,51verbose: bool = True) -> None:52desc = s53if path is not None:54os.chdir(path)55desc += " # in '%s'" % path56if verbose:57print(desc)585960def cmd(s: str,61path: Optional[str] = None,62verbose: bool = True,63noerr=False) -> None:64home: str = os.path.abspath(os.curdir)65try:66handle_path(s, path, verbose)67if os.system(s):68msg = f"Error executing '{s}'"69if noerr:70print(msg)71else:72raise RuntimeError(msg)73finally:74os.chdir(home)757677def run(s: str, path: Optional[str] = None, verbose: bool = True) -> str:78home = os.path.abspath(os.curdir)79try:80handle_path(s, path, verbose)81a = subprocess.run(s, shell=True, stdout=subprocess.PIPE)82out = a.stdout.decode('utf8')83if a.returncode:84raise RuntimeError("Error executing '%s'" % s)85return out86finally:87os.chdir(home)888990def thread_map(callable: Callable,91inputs: List[Any],92nb_threads: int = 10) -> List:93if len(inputs) == 0:94return []95if nb_threads == 1:96return [callable(x) for x in inputs]97from multiprocessing.pool import ThreadPool98tp = ThreadPool(nb_threads)99return tp.map(callable, inputs)100101102def all_packages() -> List[str]:103# Compute all the packages. Explicit order in some cases *does* matter as noted in comments,104# but we use "tsc --build", which automatically builds deps if not built.105v = [106'packages/', # top level workspace107'packages/cdn', # packages/hub assumes this is built108'packages/util',109'packages/sync',110'packages/sync-client',111'packages/sync-fs',112'packages/backend',113'packages/api-client',114'packages/jupyter',115'packages/comm',116'packages/project', # project depends on frontend for nbconvert (but NEVER vice versa again)117'packages/assets',118'packages/frontend', # static depends on frontend119'packages/static', # packages/hub assumes this is built (for webpack dev server)120'packages/server', # packages/next assumes this is built121'packages/database', # packages/next also assumes database is built (or at least the coffeescript in it is)122'packages/next',123'packages/hub', # hub won't build if next isn't already built124]125for x in os.listdir('packages'):126path = os.path.join("packages", x)127if path not in v and os.path.isdir(path) and os.path.exists(128os.path.join(path, 'package.json')):129v.append(path)130return v131132133def packages(args) -> List[str]:134v = all_packages()135# Filter to only the ones in packages (if given)136if args.packages:137packages = set(args.packages.split(','))138v = [x for x in v if x.split('/')[-1] in packages]139140# Only take things not in exclude141if args.exclude:142exclude = set(args.exclude.split(','))143v = [x for x in v if x.split('/')[-1] not in exclude]144145print("Packages: ", ', '.join(v))146return v147148149def package_json(package: str) -> dict:150return json.loads(open(f'{package}/package.json').read())151152153def write_package_json(package: str, x: dict) -> None:154open(f'{package}/package.json', 'w').write(json.dumps(x, indent=2))155156157def dependent_packages(package: str) -> List[str]:158# Get a list of the packages159# it depends on by reading package.json160x = package_json(package)161if "workspaces" not in x:162# no workspaces163return []164v: List[str] = []165for path in x["workspaces"]:166# path is a relative path167npath = os.path.normpath(os.path.join(package, path))168if npath != package:169v.append(npath)170return v171172173def get_package_version(package: str) -> str:174return package_json(package)["version"]175176177def get_package_npm_name(package: str) -> str:178return package_json(package)["name"]179180181def update_dependent_versions(package: str) -> None:182"""183Update the versions of all of the workspaces that this184package depends on. The versions are set to whatever the185current version is in the dependent packages package.json.186187There is a problem here, if you are publishing two188packages A and B with versions vA and vB. If you first publish189A, then you set it as depending on B@vB. However, when you then190publish B you set its new version as vB+1, so A got published191with the wrong version. It's thus important to first192update all the versions of the packages that will be published193in a single phase, then update the dependent version numbers, and194finally actually publish the packages to npm. There will unavoidably195be an interval of time when some of the packages are impossible to196install (e.g., because A got published and depends on B@vB+1, but B197isn't yet published).198"""199x = package_json(package)200changed = False201for dependent in dependent_packages(package):202print(f"Considering '{dependent}'")203try:204package_version = '^' + get_package_version(dependent)205except:206print(f"Skipping '{dependent}' since package not available")207continue208npm_name = get_package_npm_name(dependent)209dev = npm_name in x.get("devDependencies", {})210if dev:211current_version = x.get("devDependencies", {}).get(npm_name, '')212else:213current_version = x.get("dependencies", {}).get(npm_name, '')214# print(dependent, npm_name, current_version, package_version)215if current_version != package_version:216print(217f"{package}: {dependent} changed from '{current_version}' to '{package_version}'"218)219x['devDependencies' if dev else 'dependencies'][220npm_name] = package_version221changed = True222if changed:223write_package_json(package, x)224225226def update_all_dependent_versions() -> None:227for package in all_packages():228update_dependent_versions(package)229230231def banner(s: str) -> None:232print("\n" + "=" * 70)233print("|| " + s)234print("=" * 70 + "\n")235236237def install(args) -> None:238v = packages(args)239if v == all_packages():240print("install all packages -- fast special case")241# much faster special case242cmd("cd packages && pnpm install")243return244245# Do "pnpm i" not in parallel246for path in v:247c = "pnpm install "248if args.prod:249c += ' --prod '250cmd(c, path)251252253# Build all the packages that need to be built.254def build(args) -> None:255v = [package for package in packages(args) if needs_build(package)]256CUR = os.path.abspath('.')257258def f(path: str) -> None:259if not args.parallel and path != 'packages/static':260# NOTE: in parallel mode we don't delete or there is no261# hope of this working.262dist = os.path.join(CUR, path, 'dist')263if os.path.exists(dist):264# clear dist/ dir265shutil.rmtree(dist, ignore_errors=True)266package_path = os.path.join(CUR, path)267if args.dev and '"build-dev"' in open(268os.path.join(CUR, path, 'package.json')).read():269cmd("pnpm run build-dev", package_path)270else:271cmd("pnpm run build", package_path)272# The build succeeded, so touch a file273# to indicate this, so we won't build again274# until something is newer than this file275cmd("touch " + SUCCESSFUL_BUILD, package_path)276277if args.parallel:278thread_map(f, v)279else:280thread_map(f, v, 1)281282283def clean(args) -> None:284v = packages(args)285286if args.dist_only:287folders = ['dist']288elif args.node_modules_only:289folders = ['node_modules']290else:291folders = ['node_modules', 'dist', SUCCESSFUL_BUILD]292293paths = []294for path in v:295for x in folders:296y = os.path.abspath(os.path.join(path, x))297if os.path.exists(y):298paths.append(y)299300def f(path):301print("rm -rf '%s'" % path)302shutil.rmtree(path, ignore_errors=True)303304if (len(paths) == 0):305banner("No node_modules or dist directories")306else:307banner("Deleting " + ', '.join(paths))308thread_map(f, paths + ['packages/node_modules'], nb_threads=10)309310if not args.node_modules_only:311banner("Running 'pnpm run clean' if it exists...")312313def g(path):314# can only use --if-present with npm, but should be fine since clean is315# usually just "rm".316cmd("npm run clean --if-present", path)317318thread_map(g, [os.path.abspath(path) for path in v],319nb_threads=3 if args.parallel else 1)320321322def delete_package_lock(args) -> None:323324def f(path: str) -> None:325p = os.path.join(path, 'package-lock.json')326if os.path.exists(p):327os.unlink(p)328# See https://github.com/sagemathinc/cocalc/issues/6123329# If we don't delete node_modules, then package-lock.json may blow up in size.330node_modules = os.path.join(path, 'node_modules')331if os.path.exists(node_modules):332shutil.rmtree(node_modules, ignore_errors=True)333334thread_map(f, [os.path.abspath(path) for path in packages(args)],335nb_threads=10)336337338def pnpm(args, noerr=False) -> None:339v = packages(args)340inputs: List[List[str]] = []341for path in v:342s = 'pnpm ' + ' '.join(['%s' % x for x in args.args])343inputs.append([s, os.path.abspath(path)])344345def f(args) -> None:346# kwds to make mypy happy347kwds = {"noerr": noerr}348cmd(*args, **kwds)349350if args.parallel:351thread_map(f, inputs, 3)352else:353thread_map(f, inputs, 1)354355356def pnpm_noerror(args) -> None:357pnpm(args, noerr=True)358359360def version_check(args):361cmd("scripts/check_npm_packages.py")362363364def node_version_check() -> None:365version = int(os.popen('node --version').read().split('.')[0][1:])366if version < 14:367err = f"CoCalc requires node.js v14, but you're using node v{version}."368if os.environ.get("COCALC_USERNAME",369'') == 'user' and 'COCALC_PROJECT_ID' in os.environ:370err += '\nIf you are using https://cocalc.com, put ". /cocalc/nvm/nvm.sh" in ~/.bashrc\nto get an appropriate version of node.'371raise RuntimeError(err)372373374def pnpm_version_check() -> None:375"""376Check if the pnpm utility is new enough377"""378version = os.popen('pnpm --version').read()379if int(version.split('.')[0]) < 7:380raise RuntimeError(381f"CoCalc requires pnpm version 7, but you're using pnpm v{version}."382)383384385def main() -> None:386node_version_check()387pnpm_version_check()388389def packages_arg(parser):390parser.add_argument(391'--packages',392type=str,393default='',394help=395'(default: ""=everything) "foo,bar" means only the packages named foo and bar'396)397parser.add_argument(398'--exclude',399type=str,400default='',401help=402'(default: ""=exclude nothing) "foo,bar" means exclude foo and bar'403)404parser.add_argument(405'--parallel',406action="store_const",407const=True,408help=409'if given, do all in parallel; this will not work in some cases and may be ignored in others'410)411412parser = argparse.ArgumentParser(prog='workspaces')413subparsers = parser.add_subparsers(help='sub-command help')414415subparser = subparsers.add_parser(416'install', help='install node_modules deps for all packages')417subparser.add_argument('--prod',418action="store_const",419const=True,420help='only install prod deps (not dev ones)')421packages_arg(subparser)422subparser.set_defaults(func=install)423424subparser = subparsers.add_parser(425'build', help='build all packages for which something has changed')426subparser.add_argument(427'--dev',428action="store_const",429const=True,430help="only build enough for development (saves time and space)")431packages_arg(subparser)432subparser.set_defaults(func=build)433434subparser = subparsers.add_parser(435'clean', help='delete dist and node_modules folders')436packages_arg(subparser)437subparser.add_argument('--dist-only',438action="store_const",439const=True,440help="only delete dist directory")441subparser.add_argument('--node-modules-only',442action="store_const",443const=True,444help="only delete node_modules directory")445subparser.set_defaults(func=clean)446447subparser = subparsers.add_parser('pnpm',448help='do "pnpm ..." in each package;')449packages_arg(subparser)450subparser.add_argument('args',451type=str,452nargs='*',453default='',454help='arguments to npm')455subparser.set_defaults(func=pnpm)456457subparser = subparsers.add_parser(458'pnpm-noerr',459help=460'like "pnpm" but suppresses errors; e.g., use for "pnpm-noerr audit fix"'461)462packages_arg(subparser)463subparser.add_argument('args',464type=str,465nargs='*',466default='',467help='arguments to pnpm')468subparser.set_defaults(func=pnpm_noerror)469470subparser = subparsers.add_parser(471'version-check', help='version consistency checks across packages')472subparser.set_defaults(func=version_check)473474args = parser.parse_args()475if hasattr(args, 'func'):476args.func(args)477478479if __name__ == '__main__':480main()481482483