Jay Taylor's notes

back to listing index

notes/bq at master · sitaramc/notes · GitHub

[web search]
Original source (github.com)
Tags: programming bash unix queue task-queue job-queue github.com
Clipped on: 2021-02-24

Skip to content
master

notes / bq

Go to file
Image (Asset 2/2) alt= History
1 contributor
1 #!/bin/bash
2
3 # simple task queue; output files are in /dev/shm/bq-$USER. Uses no locks;
4 # use 'mv' command, which is atomic (within the same file system anyway) to
5 # prevent contention.
6 # ref: https://rcrowley.org/2010/01/06/things-unix-can-do-atomically.html
7
8 # run "bq -w" once to start a worker
9
10 # run "bq command [args...]" to put tasks in queue
11
12 # run "bq" to view the output directory using vifm, unless $BQ_FILEMANAGER is set
13
14 # see bq.mkd for more (e.g., using different queues, increasing/decreasing the
15 # number of workers in a queue, etc.)
16
17 # ----------------------------------------------------------------------
18
19 die() { echo "$@" >&2; exit 1; }
20
21 [ "$1" = "-h" ] && {
22 cat <<-EOF
23 Example usage:
24 # start a worker
25 bq -w
26 # submit a job
27 bq some-command arg1 arg2 [...]
28 # check status
29 bq # uses vifm as the file manager
30 export BQ_FILEMANAGER=mc; bq # env var overrides default
31 # you can only run one simple command; if you have a command with
32 # shell meta characters (;, &, &&, ||, >, <, etc), do this:
33 bq bash -c 'echo hello; echo there >junk.\$RANDOM'
34 EOF
35 exit 1
36 }
37
38 # ----------------------------------------------------------------------
39 # SETUP
40
41 TMP=/dev/shm
42 [ -d $TMP ] || TMP=/tmp
43
44 # I doubt I will ever use multiple Qs, but it's easy enough to implement
45 Q=default
46 [ "$1" = "-q" ] && {
47 [ -z "$2" ] && die "-q needs a queue name"
48 Q=$2; shift; shift
49 }
50 [ -z "$QDIR" ] && export QDIR=$TMP/bq-$USER-$Q
51 mkdir -p $QDIR/w
52 mkdir -p $QDIR/q
53 mkdir -p $QDIR/OK
54
55 # ----------------------------------------------------------------------
56 # WORK 1 TASK
57
58 _work_1() {
59 ID=$1
60
61 # "claim" the task in q by renaming q/$ID to a name that contains our own PID
62 mv q/$ID $ID.running.$$ 2>/dev/null
63
64 # if the "claim" succeeded, we won the race to run the task
65 if [ -f $ID.running.$$ ]
66 then
67 # get the command line arguments and run them
68 readarray -t cmd < $ID.running.$$
69
70 # the first line is the directory to be in; shift that out first
71 newpwd="${cmd[0]}"
72 cmd=("${cmd[@]:1}")
73
74 # log the command for debugging later
75 echo -n "cd $newpwd; ${cmd[@]}" >> w/$$
76
77 # the directory may have disappeared between submitting the
78 # job and running it now. Catch that by trying to cd to it
79 cd "$newpwd" || cmd=(cd "$newpwd")
80 # if the cd failed, we simply replace the actual command with
81 # the same "cd", and let it run and catch the error. Bit of a
82 # subterfuge, actually, but it works fine.
83
84 # finally we run the task. Note that our PWD now is NOT $QDIR, so
85 # the two redirected filenames have to be fully qualified
86 "${cmd[@]}" > $QDIR/$ID.1 2> $QDIR/$ID.2
87 ec=$?
88
89 cd $QDIR
90 mv $ID.running.$$ $ID.exitcode=$ec
91 [ "$ec" = "0" ] && mv $ID.* OK
92 echo " # $ec" >> w/$$
93 notify-send "`wc -l w/$$`" "`tail -1 w/$$`"
94 fi
95 }
96
97 # ----------------------------------------------------------------------
98 # START AND DAEMONISE A WORKER
99
100 # '-w' starts a worker; each worker runs one job at a time, so if you want
101 # more jobs to run simultaneously, run this multiple times!
102 [ "$1" = "-w" ] && [ -z "$2" ] && {
103
104 # if the user is starting a worker, any existing kill commands don't apply
105 rm -f $QDIR/q/0.*.-k
106
107 # daemonize
108 nohup "$0" -w $QDIR &
109
110 # remind the user how many workers he has started, in case he forgot
111 sleep 0.5 # wait for the other task to kick off
112 echo `cd $QDIR/w; ls | grep -v exited | wc -l` workers running
113 exit 0
114 }
115
116 # ----------------------------------------------------------------------
117 # STOP A WORKER
118
119 [ "$1" = "-k" ] && [ -z "$2" ] && {
120 touch $QDIR/q/0.$$.-k
121 # starting with a "0" assures that in an "ls" this file will come before
122 # any normal task files (which all start with `date +%s`). The contents
123 # don't matter, since it won't be "executed" in the normal manner.
124 exit 0
125 }
126
127 # ----------------------------------------------------------------------
128 # WORKER LOOP
129
130 [ "$1" = "-w" ] && {
131
132 touch $QDIR/w/$$
133
134 while :
135 do
136 cd $QDIR
137
138 ID=`cd q; ls | head -1`
139 # if nothing is waiting in q, go to sleep, but use inotifywait so you
140 # get woken up immediately if a new task lands
141 [ -z "$ID" ] && {
142 inotifywait -q -t 60 -e create q >/dev/null
143 continue
144 # whether we got an event or just timed out, we just go back round
145 }
146
147 # note there is still a bit of a race here. If tasks were submitted
148 # *between* the "ID=" and the "inotifywait" above, they will end up
149 # waiting 60 seconds before they get picked up. Hopefully that's a
150 # corner case, and anyway at worst it only causes a delay.
151
152 # handle exit, again using the "mv is atomic" principle
153 [[ $ID == 0.*.-k ]] && {
154 mv q/$ID $ID.exiting
155 [ -f $ID.exiting ] && {
156 mv w/$$ w/$$.exited
157 rm $ID.exiting
158 exit 0
159 }
160 }
161
162 # ok there was at least one task waiting; try to "work" it
163 _work_1 $ID
164 done
165
166 # we should never get here
167 touch $QDIR/worker.$$.unexpected-error
168 }
169
170 # ----------------------------------------------------------------------
171 # STATUS
172
173 # examine the output directory using $BQ_FILEMANAGER (defaulting to vifm)
174 [ -z "$1" ] && exec sh -c "${BQ_FILEMANAGER:-vifm} $QDIR"
175
176 # ----------------------------------------------------------------------
177
178 # some command was given; add it to the queue
179 ID=`date +%s`.$$.${1//[^a-z0-9_.-]/}
180 pwd > $QDIR/q/$ID
181 printf "%s\n" "$@" >> $QDIR/q/$ID