CoCalc -- rebuild_projects.py

GitHub Repository: sagemathinc/cocalc
Path: blob/master/src/scripts/rebuild_projects.py
Views: ²⁷⁵
1
###############################################################################
2
#
3
#    CoCalc: Collaborative Calculation
4
#
5
#    Copyright (C) 2016, Sagemath Inc.
6
#
7
#    This program is free software: you can redistribute it and/or modify
8
#    it under the terms of the GNU General Public License as published by
9
#    the Free Software Foundation, either version 3 of the License, or
10
#    (at your option) any later version.
11
#
12
#    This program is distributed in the hope that it will be useful,
13
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
14
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15
#    GNU General Public License for more details.
16
#
17
#    You should have received a copy of the GNU General Public License
18
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
19
#
20
###############################################################################
21

22
import argparse, hashlib, os, random, time
23
from subprocess import Popen, PIPE
24

25

26
def cmd(s):
27
    print(s)
28
    out = Popen(
29
        s, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=not isinstance(s, list))
30
    x = out.stdout.read() + out.stderr.read()
31
    e = out.wait()
32
    return x
33

34

35
def all_projects(filename='projects'):
36
    return [x.strip() for x in open(filename).readlines() if x.strip()]
37

38

39
def rebuild(project_id):
40
    if 'does not exist' not in cmd("zfs list projects-new/%s" % project_id):
41
        print("%s already done" % project_id)
42
        return
43
    if project_id in ''.join(cmd('ps ax')):
44
        print("skipping since %s is in progress, evidently" % project_id)
45
        return
46
    v = cmd("zfs list -r -t snapshot projects/%s|tail -1" % project_id).split()
47
    print(v)
48
    if len(v) < 2:
49
        return  # nothing to do
50
    try:
51
        cmd("time zfs send -Rv %s 2>>log | mbuffer -s 256k -m 8G -o - | zfs recv -vu projects-new/%s 2>>log"
52
            % (v[0], project_id))
53
    except:
54
        cmd("zfs destroy -r projects-new/%s" % project_id)
55

56

57
def rebuild_all(filename='projects', reverse=False):
58
    i = 0
59
    t0 = time.time()
60
    v = all_projects(filename=filename)
61
    if reverse:
62
        v = list(reversed(v))
63
    for p in v:
64
        i += 1
65
        print("%s / %s: %s" % (i, len(v), p))
66
        t = time.time()
67
        rebuild(p)
68
        print(time.time() - t) / 60, "   (total time=%s minutes)" % (
69
            (time.time() - t0) / 60)
70

71