DistRen Changeset - a255abd6e6f8 · protofusion repositories

Changeset - a255abd6e6f8

Parent rev.

Child rev.

[Not reviewed]

default

0 9 0

ethanzonca - 16 years ago 2009-12-06 21:51:48
ez@ethanzonca.com

Conf replacer made more flexible, conf issue fixed (causes segfault now), fixed style, updated comments

9 files changed with 85 insertions and 74 deletions:

TODO

etc/distrenclient.conf

etc/distrencommon.conf

etc/distrendaemon.conf

etc/distrenslave.conf

etc/distrenslave.conf.in

src/server/slave.c

src/server/slavefuncs.c

src/server/slavefuncs.h

0 comments (0 inline, 0 general)

TODO

➞

Show inline comments

@@ @@ -29,8 +29,8 @@ M-explain members of struct general_info @@
 Slave
  -Fix current bugs
 *-Currently distrencommon.conf is not created slave-side
 *-Add calls to remoteio, once it works or even before it works -> write message API functions and API
 ?-Other XML-writing/reading: Update exec_blender() to use some struct info, maybe. The struct is being passed to it.
 =-Make code more flexible for different types of jobs and operating systems (such as paths, libs, etc)
 =-Add code to allow pausing/resuming of the slave (including a running blender process) -> signal handling (SIGTSTP vs. SIGSTOP), distrenslave_control cmdline interface
 B-Add tmpdir variable (just like the datadir variable) that is compile-time or whatever -> simple API to allow configuration-file override while taking into account defaults set by ./configure

etc/distrenclient.conf

➞

Show inline comments

@@ @@ -5,7 +5,7 @@ @@
 */
 client
+{
-  server = "ohnopublishing"
+  server = "protofusion"
+}
 include("distrencommon.conf")

etc/distrencommon.conf

➞

Show inline comments

@@ @@ -4,16 +4,13 @@ @@
 */
 /*
   currently, server's are only supported rudimentarily. If a job has n frames and there are s servers, n / (s + 1) frames will be sent to each individual server and also rendered on the machine the job was submitted to. There will be no recursive distribution of jobs, but I want to make that possible in the future. AND, I want job IDs to have significance based on 1: the files, 2: the versions of software used to render just like git, mercurial, or bazaar's commit IDs have significance. This will allow global distribution of renderjobs without requiring central servers to coordinate the jobs - a network only need be distributed. And complex algorithms based on timeouts and completion of jobs should allow slow servers to reassign jobs to fast ones and, possibly, find shorter routes to return the resulting images to the original job submitter.
   currently, server's are only supported rudimentarily. If a job has n frames and there are s servers, n / (s + 1) frames will be sent to each individual server and also rendered on the machine the job was submitted to. There will be no recursive distribution of jobs, but I want to make that possible in the future. AND, I want job IDs to have significance based on 1: the files, 2: the versions of software used to render just like git, mercurial, or bazaar's commit IDs have significance. This will allow global distribution of renderjobs without requiring central servers to coordinate the jobs - a network only need be distributed. And complex algorithms based on timeouts and completion of jobs should allow slow servers to reassign jobs to fast ones and, possibly, find shorter routes to return the resulting images to the original job submitter. Multiple server declarations are currently included, but an implementation for multiple servers is lacking.
 */
   Just a pointer, for the multiple server architecture, we would need to designate one server as a "master" server to avoid obvious issues. We can code it flexibly though. --Normaldotcom
      I don't know what ``obvious'' issues you're talking about ;-) --ohnobinki
 */
 server protofusion
+{
   hostname = "protofusion.org"
   username = "distrenc"
   /* method's use is not implemented, ssh is the only option atm ' */
   method = "ssh"
   types = {"submit", "distribution"} /* submit means it accepts jobs, distribution means it can host files */
+}

etc/distrendaemon.conf

➞

Show inline comments

@@ @@ -3,23 +3,15 @@ @@
 */
 daemon
+{
   datadir = "/home/ohnobinki/var/distren"
 //  render_types = {"povray", "blender", "inkscape", "imagemagick", "dcraw"}
 /*
 listen {
        type = "unix"
        path = "/home/ohnobinki/var/run/distrend.sock"
        port = "0770"
   datadir = "/home/ethanzonca/var/distren"
   render_types = {"povray", "blender", "inkscape", "imagemagick", "dcraw"}
+}
 */
 listen {
        type = "tcp"
        path = "*"
        port = 4554
+}
 } /* daemon */
 include("distrencommon.conf")
 listen {
        type = "unix"
        path = "/home/ethanzonca/var/run/distrend.sock"
        port = "0770"
+}

etc/distrenslave.conf

➞

Show inline comments

 slave
+{
   username = "!username"
   datadir = "/home/ethanzonca/var/distren"
+}
 include("distrencommon.conf")
 datadir = "/home/ohnobinki/var/distren"

etc/distrenslave.conf.in

➞

Show inline comments

 slave
+{
   username = "!username"
   datadir = "@LOCALSTATEDIR@/@PACKAGE@"
+}
 include("distrencommon.conf")
 datadir = "@LOCALSTATEDIR@/@PACKAGE@"
 include("distrencommon.conf")
@@ \ No newline at end of file @@

src/server/slave.c

➞

Show inline comments

@@ @@ -33,18 +33,35 @@ @@
 int main(int argc, char *argv[])
+{
   int counter;
   char curopt;
   char *username;
   /* Parse arguments */
   for(counter = 0; counter < argc; counter ++)
     if(strcmp(argv[counter], "-h") == 0)
+      {
 	fprintf(stderr, "Usage: distrenslave [option] \nStarts a distren slave\n\t-h\tshow this help\n");
 	return 2;
+      }
   while(-1 != (curopt = getopt(argc, argv, "u:h")))
+     {
        if(curopt == ':')
+         {
            fprintf(stderr, "-%c: is missing an argument\n", optopt);
            return 1;
+         }
        else if(curopt == '?')
+         {
            fprintf(stderr, "-%c: invalid option specified\n", optopt);
            return 1;
+         }
        else if(curopt == 'h')
+         {
            fprintf(stderr, "Usage: distrenslave [option] \nStarts a distren slave\n\t-u\tset username (run after fresh install)\n\t-h\tshow this help\n");
            return 2;
+         }
        else if(curopt == 'u')
          username = strdup(optarg);
          fprintf(stderr, "Putting username \"%s\" in distrenslave.conf\n", username);
          conf_replace("distrenslave.conf", "!username", username);
          fprintf(stderr, "Please invoke distrenslave with no arguments to run with the username you just set\n");
          return 0;
+     }
   /* Get conf data */
   char *username;
   char *datadir;
   cfg_t * my_cfg;
   cfg_opt_t myopts[] = {
@@ @@ -103,6 +120,7 @@ int main(int argc, char *argv[]) @@
+    {
     tell_the_server(0); // Give me some work!
     /* If we got a frame */
     if(haveWork)
+      {
@@ @@ -133,17 +151,17 @@ int main(int argc, char *argv[]) @@
+            }
             else {
               fprintf(stderr, "Downloading job data from server failed. Check your network connection.\n");
-              return 1; // for now
+              return 1; // Eventually make a retry loop
+            }
+          }
         else
           fprintf(stderr, "Using cached job file...\n");
         _distren_asprintf(&outdir, "/tmp/distren/job%d", jobnum); /*< @TODO: free() */
         mkdir("/tmp/distren", 0750); /* @TODO: Make this less *nix-specific */
         _distren_asprintf(&outdir, "/tmp/distren/job%d", jobnum);
         mkdir("/tmp/distren", 0750); /* @TODO: Change to tmpdir once it exists */
         mkdir(outdir, 0750);
-        _distren_asprintf(&tarcmd, "tar -xvf \"%s\" -C \"%s\"", pathtoTar, outdir); /* @TODO: Make this portable. Libtar or something? */
+        _distren_asprintf(&tarcmd, "tar -xvf \"%s\" -C \"%s\"", pathtoTar, outdir); /* @TODO:Use a lib here! */
         system(tarcmd);
         free(tarcmd);
         free(pathtoTar);
@@ @@ -165,34 +183,38 @@ int main(int argc, char *argv[]) @@
+          }
         /* Variable-fillers which require XML */
-        outputExt = myjob->output_format; /* @TODO: FIXME! <-- I WILL! */
         outputExt = myjob->output_format;
         /* Prepares the path to the jobfile */
         _distren_asprintf(&pathtoOutput, "%s/job%d/output/job%d-frame%d.%s", datadir, jobnum, jobnum, framenum, outputExt );
         free(outputExt);
         /* Execute blender */
         if(exec_blender(pathtoJobfile, pathtoOutput, framenum)){ /* @TODO: This warning should be fixed :D */
         if(exec_blender(pathtoJobfile, pathtoOutput, framenum))
+          {
             fprintf(stderr,"Error running Blender. Check your installation and/or your PATH.");
             return 1;
+        }
+          }
         free(pathtoJobfile);
-        /* When blender is finished, run this... */
+        /* Post-execution */
         fprintf(stderr, "Finished frame %d in job %d, uploading...", framenum, jobnum);
         if( !curlpost(pathtoOutput, urltoOutput)){ // Uploads output
           fprintf(stderr,"Upload successful, removing old output...\n");
           remove(pathtoOutput); // Delete the file after its uploaded
+        }
         else{
           fprintf(stderr,"Upload failed. Check your network connection. Trying again...");
           int tries=0;
           while(tries<10 && curlpost(pathtoOutput, urltoOutput)){
               fprintf(stderr, "Upload failed. Trying again in 10 seconds... (attempt %d of 10)", tries);
               tries++;
               sleep(10);
         if( !curlpost(pathtoOutput, urltoOutput)) // Uploads output
+          {
             fprintf(stderr,"Upload successful, removing old output...\n");
             remove(pathtoOutput); // Delete the file after its uploaded
+          }
           // @TODO: Keep track of files that we were unable to upload, and upload them later
+        }
         else
+          {
             fprintf(stderr,"Upload failed. Check your network connection. Trying again...");
             int tries=0;
             while(tries<10 && curlpost(pathtoOutput, urltoOutput))
+              {
                 fprintf(stderr, "Upload failed. Trying again in 10 seconds... (attempt %d of 10)", tries);
                 tries++;
                 sleep(10);
+              }
             // @TODO: Keep track of files that we were unable to upload, and upload them later
+          }
         free(urltoOutput);
         free(pathtoOutput);
@@ @@ -202,11 +224,12 @@ int main(int argc, char *argv[]) @@
       fprintf(stderr,"Nothing to do. Idling...\n");
     // If the server says that every frame for the last jobnum is finished, OR if the data is getting old
     if(1 == 0){
       // Note: individual frames are already deleted after uploading,
       // except for ones that couldn't be uploaded
       delete_jobdata(jobnum, datadir);
+    }
     if(1 == 0)
+      {
         // Note: individual frames are already deleted after uploading,
         // except for ones that couldn't be uploaded
         delete_jobdata(jobnum, datadir);
+      }
     sleep(5); // Poll 5 seconds. @TODO: Remove all polling
+  }

src/server/slavefuncs.c

➞

Show inline comments

@@ @@ -167,17 +167,17 @@ int login_user(char *username) @@
   return 1; // success
+}
 /** Replaces !username with a username in the slave's conf file */
 int conf_replace(char *username){
 /** Replaces wordtoreplace with replacewith in conffile (relative to SYSCONFDIR) */
 int conf_replace(char *conffile, char *wordtoreplace, char *replacewith){
   int maxlinelen = 120;
   char *fileOrig = SYSCONFDIR "/distrenslave.conf";
   char *fileRepl = SYSCONFDIR "/distrenslave.conf.edited";
   char *text2find = "!username";
   char *text2repl = username;
   char *fileOrig;
   char *fileRepl;
   _distren_asprintf(&fileOrig, "&s&s", SYSCONFDIR, conffile);
   _distren_asprintf(&fileRepl, "&s&s.edited", SYSCONFDIR, conffile);
   char buffer[maxlinelen+2];
   char *buff_ptr, *find_ptr;
   FILE *fp1, *fp2;
-  size_t find_len = strlen(text2find);
+  size_t find_len = strlen(wordtoreplace);
   fp1 = fopen(fileOrig,"r");
   fp2 = fopen(fileRepl,"w");
   if (fp1 ==NULL){
@@ @@ -192,11 +192,11 @@ int conf_replace(char *username){ @@
     while(fgets(buffer,maxlinelen+2,fp1))
+      {
         buff_ptr = buffer;
-        while ((find_ptr = strstr(buff_ptr,text2find)))
+        while ((find_ptr = strstr(buff_ptr,wordtoreplace)))
+        {
            while(buff_ptr < find_ptr)
              fputc((int)*buff_ptr++,fp2);
-           fputs(text2repl,fp2);
+           fputs(replacewith,fp2);
            buff_ptr += find_len;
+         }
          fputs(buff_ptr,fp2);
@@ @@ -219,8 +219,8 @@ int exec_blender(char *input, char *outp @@
   int ret;
   char *frame_str;
   char *command = "blender"; // @TODO: append .exe if win32? Do we expect this to be in the PATH?
   char *cmd[] = { command, "-b", "-o", output, input, "-f", frame_str, (char *)NULL };
   char *command = "blender"; // @TODO: We currently expect this to be in PATH
   char *cmd[] = { command, "-b", "-o", output, input, "-f", frame_str, (char *)NULL }; // arguments for blender
   char buf[10];
   struct execio *testrem;
@@ @@ -228,6 +228,7 @@ int exec_blender(char *input, char *outp @@
   _distren_asprintf(&frame_str, "%i", frame);
   fprintf(stderr,"Executing: %s\n", frame_str);
   ret = execio_open(&testrem, command, cmd);
   buf[9] = '\0';
   while(!execio_read(testrem, buf, 9, &readlen))

src/server/slavefuncs.h

➞

Show inline comments

@@ @@ -35,7 +35,7 @@ int curlpost(char *filename, char *url); @@
 int ssh_keygen();
 int register_user(char *username, char *email);
 int login_user(char *username);
-int conf_replace(char *username);
+int conf_replace(char *conffile, char *wordtoreplace, char *replacewith);
 int exec_blender(char *input, char *output, int frame);
 void xmlinit();
 void xmlcleanup();

0 comments (0 inline, 0 general)