Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place. Commercial Alternative to JupyterHub.
Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place. Commercial Alternative to JupyterHub.
Path: blob/master/src/packages/util/db-schema/blobs.ts
Views: 923
/*1* This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.2* License: MS-RSL – see LICENSE.md for details3*/45import { Table } from "./types";67// Note that github has a 10MB limit --8// https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/attaching-files9// All code in cocalc (frontend, etc.) should use this,10// rather than copying or defining their own!11export const MAX_BLOB_SIZE = 10000000;1213// some throttling -- note that after a bit, most blobs end up longterm14// cloud storage and are never accessed. This is mainly a limit to15// prevent abuse.16export const MAX_BLOB_SIZE_PER_PROJECT_PER_DAY = {17licensed: 100 * MAX_BLOB_SIZE,18unlicensed: 10 * MAX_BLOB_SIZE,19};2021Table({22name: "blobs",23fields: {24id: {25type: "uuid",26desc: "The uuid of this blob, which is a uuid derived from the Sha1 hash of the blob content.",27},28blob: {29type: "Buffer",30desc: "The actual blob content",31},32expire: {33type: "timestamp",34desc: "When to expire this blob (when delete_expired is called on the database).",35},36created: {37type: "timestamp",38desc: "When the blob was created.",39},40project_id: {41// I'm not really sure why we record a project associated to the blob, rather42// than something else (e.g., account_id)-- update: added that. However, it's useful for abuse, since43// if abuse happened with a project, we could easily delete all corresponding blobs,44// and also it's a good tag for throttling.45type: "string",46desc: "The uuid of the project that created the blob, if it is associated to a project.",47},48account_id: {49type: "uuid",50desc: "The uuid of the account that created the blob. (Only started recording in late 2024. Will make it so a user can optionally delete any blobs associated to their account when deleting their account.)",51},52last_active: {53type: "timestamp",54desc: "When the blob was last pulled from the database.",55},56count: {57type: "number",58desc: "How many times the blob has been pulled from the database.",59},60size: {61type: "number",62desc: "The size in bytes of the blob.",63},64gcloud: {65type: "string",66desc: "name of a bucket that contains the actual blob, if available.",67},68backup: {69type: "boolean",70desc: "if true, then this blob was saved to an offsite backup",71},72compress: {73type: "string",74desc: "optional compression used: 'gzip' or 'zlib'",75},76},77rules: {78desc: "Table that stores blobs mainly generated as output of Sage worksheets.",79primary_key: "id",80// these indices speed up the search been done in 'copy_all_blobs_to_gcloud'81// less important to make this query fast, but we want to avoid thrashing cache82pg_indexes: ["((expire IS NULL))", "((gcloud IS NULL))", "last_active"],83user_query: {84get: {85async instead_of_query(database, opts, cb): Promise<void> {86const obj: any = Object.assign({}, opts.query);87if (obj == null || obj.id == null) {88cb("id must be specified");89return;90}91database.get_blob({92uuid: obj.id,93cb(err, blob) {94if (err) {95cb(err);96} else {97cb(undefined, { id: obj.id, blob });98}99},100});101},102fields: {103id: null,104blob: null,105},106},107set: {108// NOTE: we put "as any" for fields below because ttl is not an actual field but109// it is allowed for set queries and determine the expire field. I would rather110// do this (which *is* supported by the backend) then not restrict the fields keys111// for other schema entries. Alternatively, we could have a special kind of field112// above that is "virtual", but that requires writing more code in the backend. We'll113// do that if necessary.114fields: {115id: true,116blob: true,117project_id: "project_write",118account_id: "account_id",119ttl: 0,120} as any,121required_fields: {122id: true,123blob: true,124project_id: true,125},126async instead_of_change(127database,128_old_value,129new_val,130account_id,131cb,132): Promise<void> {133database.save_blob({134uuid: new_val.id,135blob: new_val.blob,136ttl: new_val.ttl,137project_id: new_val.project_id,138account_id,139check: true, // can't trust the user!140cb,141});142},143},144},145},146});147148149