Changeset - 78516a1d1aee
[Not reviewed]
default
0 4 0
Ethan Zonca (ethanzonca) - 16 years ago 2010-03-14 16:39:49
e@ethanzonca.com
Distren is now functional
4 files changed with 67 insertions and 17 deletions:
0 comments (0 inline, 0 general)
src/server/simpleslave.c
Show inline comments
 
@@ -66,158 +66,166 @@ int main(int argc, char *argv[])
 
         {
 
           fprintf(stderr, "-%c: is missing an argument\n", optopt);
 
           return 1;
 
         }
 
       else if(curopt == '?')
 
         {
 
           fprintf(stderr, "-%c: invalid option specified\n", optopt);
 
           return 1;
 
         }
 
       else if(curopt == 'h')
 
         {
 
           fprintf(stderr, "Usage: distrenslave [option] \nStarts a distren slave\n\t-u\tset username (run after fresh install)\n\t-t\tEnter test mode\n\t-h\tshow this help\n");
 
           return 2;
 
         }
 
       else if(curopt == 't')
 
         {
 
           slaveTest();
 
           return 0;
 
         }
 
       else if(curopt == 'u')
 
         username = strdup(optarg);
 
         fprintf(stderr, "Putting username \"%s\" in distrenslave.conf\n", username);
 
         conf_replace("distrenslave.conf", "!username", username);
 
         fprintf(stderr, "Please invoke distrenslave with no arguments to run with the username you just set\n");
 
         return 0;
 
     }
 

	
 
  /* Get conf data */
 
  options_init(argc, argv, &my_cfg, myopts, "slave", &commonopts);
 

	
 
  /* Notifies the user if there no username in .conf */
 
  if(checkUsername(username))
 
    return 1;
 
  if(!strncmp(password, "!password",10))
 
    {
 
      fprintf(stderr, "You haven't specified a password. Please edit your distrenslave.conf file appropriately\n");
 
      return 1;
 
    }
 

	
 
  // Variables needed for main loop
 
  int jobnum = 0;
 
  int framenum = 0;
 
  int slavekey = atoi(username); // @TODO: Make this more friendly
 

	
 
  char *urltoTar;      /* Full URL to the server-side location of job#.tgz */
 
  char *pathtoTar;     /* Full path to the location of the job#.tgz */
 
  char *pathtoTardir;
 

	
 
  // Temporary for uncomressed testing
 
  char *urltoJobfile;
 

	
 
  char *urltoOutput;   /* Full URL where output is posted */
 
  char *pathtoOutput;  /* Full path to the output (rendered) file */
 
  char *pathtoOutdir;  /* Full path to output directory */
 
  char *pathtoRenderOutput; /* Contains blender framenum placeholder */
 

	
 
  char *pathtoJob; /* Path to job data folder */
 
  char *pathtoJobfile; /* Full path to the job's main file */
 
  char *outputExt = "jpg";     /* Output Extension (e.g., JPG) */
 

	
 
  int haveWork = 0;
 
  int quit = 0;
 

	
 
  // Main loop
 
  while(!quit)
 
    {
 

	
 
    // request work
 
    fprintf(stderr,"Requesting work...\n");
 
    haveWork = _web_getwork(slavekey, password, &jobnum, &framenum);
 

	
 
    /* If we got a frame */
 
    if(haveWork)
 
      {
 
        fprintf(stderr,"Got work from server...\n");
 
        /* @TODO: Add remotio hooks */
 
        // jobnum = remoteio_read(jobnum); /* Set jobnum from remoteio (we could use info from struct, but we need this info to download the xmlfile */
 
        // framenum = remoteio_read(jobnum); /* Set framenum from remoteio */
 
        // outputExt = remotio)read(outputExt); /* Set output extension from remotio */
 

	
 
        fprintf(stderr, "Preparing to render frame %d in job %d\n", framenum, jobnum);
 

	
 
        prepareJobPaths(jobnum, framenum, outputExt, datadir, &urltoTar, &pathtoTar, &pathtoTardir, &pathtoJob, &pathtoJobfile, &urltoOutput, &pathtoOutput, &pathtoOutdir);
 
        prepareJobPaths(jobnum, framenum, outputExt, datadir, &urltoTar, &pathtoTar, &pathtoTardir, &pathtoJob, &pathtoJobfile, &urltoJobfile, &urltoOutput, &pathtoOutput, &pathtoRenderOutput, &pathtoOutdir);
 
        //free(outputExt);
 
        mkdir(pathtoTardir, 0700);
 
        int dlret = downloadTar(urltoTar, pathtoTar);
 
        // int dlret = downloadTar(urltoTar, pathtoTar);
 
        int dlret = downloadTar(urltoJobfile, pathtoJobfile);
 
        if(dlret == 0)
 
          fprintf(stderr,"Got data tarball\n");
 
          // fprintf(stderr,"Got data tarball\n");
 
          fprintf(stderr,"Got uncompressed data!\n");
 
        else if(dlret == 3){
 
          _web_resetframe(slavekey, password, jobnum, framenum);  // Unassign the frame on the server so other slaves can render it
 
          return 0; // ouput dir doesn't exist
 
        }
 
        else
 
          fprintf(stderr,"Using existing tarball %s...\n", pathtoTar);
 
          fprintf(stderr,"Using existing uncompressed data %s...\n", pathtoJobfile);
 
          //fprintf(stderr,"Using existing tarball %s...\n", pathtoTar);
 

	
 
        /* // Commented for decompression
 
        struct stat jbuffer;
 
        int jstatus = stat(pathtoJobfile, &jbuffer);
 
        if(jstatus == -1){
 
          fprintf(stderr,"Main job file does not exist, extracting...\n");
 

	
 
          // If error unpacking tarball
 
          if(unpackJob(pathtoJob, pathtoTar)){
 
            _web_resetframe(slavekey, password, jobnum, framenum);  // Unassign the frame on the server so other slaves can render it
 
            return 1;
 
          }
 
        }
 

	
 
        */
 

	
 
        /* ignore return because directory may exist already */
 
        fprintf(stderr,"Creating output directory %s\n", pathtoOutdir);
 
        mkdir(pathtoOutdir, 0700);
 
        fprintf(stderr,"Marking frame started on server... ");
 
        _web_startframe(slavekey, password, jobnum, framenum);
 

	
 
        /* Execute blender */
 
        fprintf(stderr,"Executing blender on file %s\n", pathtoJobfile);
 
        fprintf(stderr,"Directing output to file %s\n", pathtoOutput);
 

	
 
        if(exec_blender(pathtoJobfile, pathtoOutput, framenum))
 
        if(exec_blender(pathtoJobfile, pathtoRenderOutput, framenum))
 
          {
 
            fprintf(stderr,"Error running Blender. Check your installation and/or your PATH.\n");
 
            _web_resetframe(slavekey, password, jobnum, framenum);  // Unassign the frame on the server so other slaves can render it
 
            return 1;
 
          }
 
        free(pathtoJobfile);
 

	
 
        struct stat buffer;
 
        int fstatus = stat(pathtoOutput, &buffer);
 
        if(fstatus == -1){
 
          fprintf(stderr,"*** Frame was not rendered correctly! Scene may not have camera, or your blender installation is not working.\n");
 
          fprintf(stderr,"*** %s doesn't exist! Scene may not have camera, or your blender installation is not working.\n", pathtoOutput);
 
          _web_resetframe(slavekey, password, jobnum, framenum);  // Unassign the frame on the server so other slaves can render it
 
          return 1;
 
        }
 
        else{
 
          /* Post-execution */
 
          fprintf(stderr, "Finished frame %d in job %d, uploading...\n", framenum, jobnum);
 
          uploadOutput(pathtoOutput, urltoOutput, jobnum, framenum, slavekey); // @TODO: Handle return value
 

	
 
          free(urltoOutput);
 
          free(pathtoOutput);
 

	
 
          // Tell the server that rendering and upload are complete
 
          _web_finishframe(slavekey, password, jobnum, framenum);
 
        }
 
      }
 
    else
 
      fprintf(stderr,"Nothing to do. Idling...\n");
 

	
 
    // @TODO: If the server says that every frame for the last jobnum is finished, OR if the data is getting old
 
    if(1 == 0)
 
      {
 
        // Note: individual frames are already deleted after uploading,
 
        // except for ones that couldn't be uploaded
 
        delete_jobdata(jobnum, datadir);
 
      }
 

	
 
    sleep(5); // Poll 5 seconds. @TODO: Remove all polling
 
  }
 

	
 
  fprintf(stderr,"Goodbye!\n");
 
  return 0;
 
}
src/server/slave.c
Show inline comments
 
@@ -71,127 +71,131 @@ int main(int argc, char *argv[])
 
           return 1;
 
         }
 
       else if(curopt == '?')
 
         {
 
           fprintf(stderr, "-%c: invalid option specified\n", optopt);
 
           return 1;
 
         }
 
       else if(curopt == 'h')
 
         {
 
           fprintf(stderr, "Usage: distrenslave [option] \nStarts a distren slave\n\t-u\tset username (run after fresh install)\n\t-t\tEnter test mode\n\t-h\tshow this help\n");
 
           return 2;
 
         }
 
       else if(curopt == 't')
 
         {
 
           slaveTest();
 
           return 0;
 
         }
 
       else if(curopt == 'u')
 
         username = strdup(optarg);
 
         fprintf(stderr, "Putting username \"%s\" in distrenslave.conf\n", username);
 
         conf_replace("distrenslave.conf", "!username", username);
 
         fprintf(stderr, "Please invoke distrenslave with no arguments to run with the username you just set\n");
 
         return 0;
 
     }
 

	
 
  /* Get conf data */
 
  options_init(argc, argv, &my_cfg, myopts, "slave", &commonopts);
 

	
 

	
 
  /* Notifies the user if there no username in .conf */
 
  if(checkUsername(username))
 
    return 1;
 

	
 
  fprintf(stderr, "Connecting to server...\n");
 
  if(remoteio_open(&comm_slave, commonopts->remoteio, server))
 
    {
 
      fprintf(stderr, "Error connecting to server; exiting\n");
 
      return 1;
 
    }
 

	
 

	
 
  // Variables needed for main loop
 
  int jobnum = 0;
 
  int framenum = 0;
 
  int slavekey = atoi(username); // @TODO: Make this more friendly
 
  char *urltoTar;      /* Full URL to the server-side location of job#.tgz */
 
  char *pathtoTar;     /* Full path to the location of the job#.tgz */
 
  char *pathtoTardir;
 
  char *pathtoRenderOutput;
 

	
 
  char *urltoOutput;   /* Full URL where output is posted */
 
  char *pathtoOutput;  /* Full path to the output (rendered) file */
 
  char *pathtoOutdir;  /* Full path to output directory */
 

	
 
  // Temporary for uncomressed testing
 
  char *urltoJobfile;
 

	
 
  char *pathtoJob;     /* Full path to job data folder */
 
  char *pathtoJobfile; /* Full path to the job's main file */
 
  char *outputExt;     /* Output Extension (e.g., JPG) */
 

	
 
  int haveWork = 0;
 
  int quit = 0;
 

	
 
  // Main loop
 
  while(!quit)
 
    {
 

	
 
    // request work
 
    fprintf(stderr,"Requesting work...\n");
 
    sendSignal(comm_slave, DISTREN_REQUEST_GETWORK);
 

	
 
    /* If we got a frame */
 
    if(haveWork)
 
      {
 
        fprintf(stderr,"Got work from server...\n");
 
        /* @TODO: Add remotio hooks */
 
        // jobnum = remoteio_read(jobnum); /* Set jobnum from remoteio (we could use info from struct, but we need this info to download the xmlfile */
 
        // framenum = remoteio_read(jobnum); /* Set framenum from remoteio */
 
        // outputExt = remotio)read(outputExt); /* Set output extension from remotio */
 

	
 
        fprintf(stderr, "Preparing to render frame %d in job %d\n", framenum, jobnum);
 

	
 
        prepareJobPaths(jobnum, framenum, outputExt, datadir, &urltoTar, &pathtoTar,&pathtoTardir,&pathtoJob, &pathtoJobfile, &urltoOutput, &pathtoOutput, &pathtoOutdir);
 
        prepareJobPaths(jobnum, framenum, outputExt, datadir, &urltoTar, &pathtoTar,&pathtoTardir,&pathtoJob, &pathtoJobfile, &urltoJobfile, &urltoOutput, &pathtoOutput, &pathtoRenderOutput, &pathtoOutdir);
 
        free(outputExt);
 

	
 
        if(downloadTar(urltoTar, pathtoTar))
 
          return 1;
 

	
 
        unpackJob(datadir, pathtoTar);
 

	
 
        /* Execute blender */
 
        if(exec_blender(pathtoJobfile, pathtoOutput, framenum))
 
          {
 
            fprintf(stderr,"Error running Blender. Check your installation and/or your PATH.\n");
 
            return 1;
 
          }
 
        free(pathtoJobfile);
 

	
 
        /* Post-execution */
 
        fprintf(stderr, "Finished frame %d in job %d, uploading...\n", framenum, jobnum);
 
        uploadOutput(pathtoOutput, urltoOutput, jobnum, framenum, slavekey); // @TODO: Handle return value
 

	
 
        free(urltoOutput);
 
        free(pathtoOutput);
 

	
 
        // Tell the server that rendering and upload are complete of "jobjum.framenum"
 
	char *jobFrame;
 
	_distren_asprintf(&jobFrame, "%s.%s", jobnum, framenum);
 
        sendExtSignal(comm_slave, DISTREN_REQUEST_DONEFRAME, jobFrame);
 
	free(jobFrame);
 
      }
 
    else
 
      fprintf(stderr,"Nothing to do. Idling...\n");
 

	
 
    // @TODO: If the server says that every frame for the last jobnum is finished, OR if the data is getting old
 
    if(1 == 0)
 
      {
 
        // Note: individual frames are already deleted after uploading,
 
        // except for ones that couldn't be uploaded
 
        delete_jobdata(jobnum, datadir);
 
      }
 

	
 
    sleep(5); // Poll 5 seconds. @TODO: Remove all polling
 
  }
 

	
 
  fprintf(stderr,"Closing connection to server...\n");
 
  remoteio_close(comm_slave);
 
  fprintf(stderr,"Goodbye!\n");
 
  return 0;
 
}
src/server/slavefuncs.c
Show inline comments
 
@@ -122,415 +122,431 @@ xmlNodePtr xml_quickxpath(xmlXPathContex
 

	
 

	
 
/** Stub for deleting job data from the disk. @TODO: unstubify me! */
 
int delete_jobdata(int jobnum, char *datadir)
 
{
 
  char *jobpath;
 
  _distren_asprintf(&jobpath, "%s/%d", datadir, jobnum);
 
  // rmdir(jobpath);
 
  fprintf(stderr, "Please manually remove %s. Automatic removal is currently not implemented.\n", jobpath);
 
  return 0;
 
}
 

	
 
/** Function referenced by curlget() to write data to disk. */
 
size_t curl_writetodisk(void *ptr, size_t size, size_t nmemb, FILE *stream)
 
 {
 
    return fwrite(ptr, size, nmemb, stream);
 
  }
 

	
 
/** Helper function for cURL's progress display */
 
int curl_progress( char *Bar,double t,double d,double ultotal,double ulnow)
 
{
 
fprintf(stderr,"Downloading: %f%% complete\r",d/t*100);
 
return 0;
 
}
 

	
 
/** Retrieves a URL with cURL and saves it to disk */
 
int curlget(char *url, char *out){
 
  fprintf(stderr,"Preparing to download %s\n",url);
 
  double *Bar; // Stores cURL progress display info
 
  CURL *curl;
 
  CURLcode res;
 
  FILE *outfile;
 

	
 
  curl = curl_easy_init();
 
  if(curl) {
 
    outfile = fopen(out, "w"); // Open where we're writing to
 
    if(outfile == NULL)
 
      return 2; // Output dir doesn't exist
 
    curl_easy_setopt(curl, CURLOPT_URL, url);
 
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, outfile);
 
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_writetodisk); // this MUST be set for win32 compat.
 
    curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
 
    curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, curl_progress);
 
    curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &Bar);
 
    res = curl_easy_perform(curl);
 
    curl_easy_cleanup(curl);
 
  }
 
  fprintf(stderr,"\n"); // Clears out he progressbar's carriage return
 
  fclose(outfile);
 
  return res; // 0 is OK, 1 is 404 or other error
 
}
 

	
 
/** Posts a file to a url with cUrl */
 
int curlpost(char *filename, char *url, int jobnum, int framenum, int slavekey){
 
  fprintf(stderr,"Uploading to %s\n", url);
 
  char *targetname = "uploadedfile"; // Name of the target in the php file on the server (Don't change me unless you have different PHP code)
 
  CURL *curl;
 
  CURLcode res;
 
  struct curl_httppost *formpost=NULL;
 
  struct curl_httppost *lastptr=NULL;
 
  struct curl_slist *headerlist=NULL;
 
  static const char buf[] = "Expect:";
 

	
 
  char *sjobnum;
 
  char *sframenum;
 
  char *sslavekey;
 
  _distren_asprintf(&sjobnum,"%d",jobnum);
 
  _distren_asprintf(&sframenum,"%d",framenum);
 
  _distren_asprintf(&sslavekey,"%d",slavekey);
 

	
 
  curl_global_init(CURL_GLOBAL_ALL);
 

	
 
  /* jobnum field... */
 
   curl_formadd(&formpost,
 
                &lastptr,
 
                CURLFORM_COPYNAME, "jobnum",
 
                CURLFORM_COPYCONTENTS, jobnum, CURLFORM_END);
 
                CURLFORM_COPYCONTENTS, sjobnum, CURLFORM_END);
 
  /* framenum field... */
 
  curl_formadd(&formpost,
 
               &lastptr,
 
               CURLFORM_COPYNAME, "framenum",
 
               CURLFORM_COPYCONTENTS, framenum, CURLFORM_END);
 
               CURLFORM_COPYCONTENTS, sframenum, CURLFORM_END);
 
  /* slavekey field... */
 
  curl_formadd(&formpost,
 
               &lastptr,
 
               CURLFORM_COPYNAME, "slavekey",
 
               CURLFORM_COPYCONTENTS, slavekey, CURLFORM_END);
 
               CURLFORM_COPYCONTENTS, sslavekey, CURLFORM_END);
 
  /* upload field... */
 
  curl_formadd(&formpost,
 
               &lastptr,
 
               CURLFORM_COPYNAME, targetname,
 
               CURLFORM_FILE, filename,
 
               CURLFORM_END);
 
  /* filename field... */
 
  curl_formadd(&formpost,
 
               &lastptr,
 
               CURLFORM_COPYNAME, "filename",
 
               CURLFORM_COPYCONTENTS, filename,
 
               CURLFORM_END);
 
  /* submit field, not usually needed, just in case... */
 
  curl_formadd(&formpost,
 
               &lastptr,
 
               CURLFORM_COPYNAME, "submit",
 
               CURLFORM_COPYCONTENTS, "send",
 
               CURLFORM_END);
 

	
 
  curl = curl_easy_init();
 
  headerlist = curl_slist_append(headerlist, buf);
 
  if(curl) {
 
    /* Setting the URL to get the post, and the contents of the post */
 
    curl_easy_setopt(curl, CURLOPT_URL, url);
 
    curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
 
    res = curl_easy_perform(curl);
 

	
 
    curl_easy_cleanup(curl);
 
    /* cleanup the formpost junk */
 
    curl_formfree(formpost);
 
    curl_slist_free_all (headerlist);
 
  }
 
  return res;
 
}
 

	
 

	
 
/** Logs the user into the server after ensuring that keys exist */
 
int login_user(char *username)
 
{
 
  // @TODO: Put some telnet-style auth code here unless this is obselete
 
  return 1; // success
 
}
 

	
 
/** Replaces wordtoreplace with replacewith in conffile (relative to SYSCONFDIR) */
 
int conf_replace(char *conffile, char *wordtoreplace, char *replacewith){
 
  int maxlinelen = 120;
 
  char *fileOrig;
 
  char *fileRepl;
 
  _distren_asprintf(&fileOrig, "%s/%s", SYSCONFDIR, conffile);
 
  _distren_asprintf(&fileRepl, "%s%s.edited", SYSCONFDIR, conffile);
 
  char buffer[maxlinelen+2];
 
  char *buff_ptr, *find_ptr;
 
  FILE *fp1, *fp2;
 
  size_t find_len = strlen(wordtoreplace);
 
  fp1 = fopen(fileOrig,"r");
 
  fp2 = fopen(fileRepl,"w");
 
  if (fp1 ==NULL){
 
    fprintf(stderr, "%s doesn't exist\n",fileOrig);
 
    return 0;
 
  }
 
  else if(fp2 ==NULL){
 
    fprintf(stderr, "Can't write a file to disk! Check permissions.\n");
 
    return 0;
 
  }
 
  else{
 
    while(fgets(buffer,maxlinelen+2,fp1))
 
      {
 
        buff_ptr = buffer;
 
        while ((find_ptr = strstr(buff_ptr,wordtoreplace)))
 
        {
 
           while(buff_ptr < find_ptr)
 
             fputc((int)*buff_ptr++,fp2);
 
           fputs(replacewith,fp2);
 
           buff_ptr += find_len;
 
         }
 
         fputs(buff_ptr,fp2);
 
      }
 
    rename(fileRepl, fileOrig);
 
  }
 
  fclose(fp2);
 
  fclose(fp1);
 
  fprintf(stderr,"Wrote conf file...\n");
 
return 1; // Success
 
}
 

	
 

	
 
/* Executors */
 

	
 
/** Executor function for Blender operations */
 
int exec_blender(char *input, char *output, int frame)
 
{
 
  int ret;
 
  char *frame_str;
 
  _distren_asprintf(&frame_str, "%i", frame);
 

	
 
  char *command = "blender"; // @TODO: We currently expect this to be in PATH
 
  char *cmd[] = { command, "-b", "-o", output, input, "-f", frame_str, (char *)NULL }; // arguments for blender
 
  char *cmd[] = { command, "-b", input, "-o", output, "-f", frame_str, (char *)NULL }; // arguments for blender
 

	
 
  fprintf(stderr,"Preparing to execute command: %s -b -o %s %s -f %s\n", command, output, input, frame_str);
 
  fprintf(stderr,"Preparing to execute command: %s -b %s -o %s -f %s\n", command, input, output, frame_str);
 

	
 
  char buf[10];
 
  struct execio *testrem;
 
  size_t readlen;
 

	
 
  fprintf(stderr,"Executing: %s\n", frame_str);
 
  ret = execio_open(&testrem, command, cmd);
 
  buf[20] = '\0';
 
  while(!execio_read(testrem, buf, 20, &readlen))
 
    {
 
      if(readlen > 20) {
 
        fprintf(stderr, "Something is terribly wrong!\n");
 
       }
 
       buf[readlen] = '\0';
 
       fprintf(stderr, "read \"%s\"\n", buf);
 
    }
 
  execio_close(testrem);
 

	
 
  return ret;
 
}
 

	
 
void xmlinit()
 
{
 
  xmlInitParser();
 
  xmlXPathInit();
 
}
 

	
 
void xmlcleanup()
 
{
 
  xmlCleanupParser();
 
}
 

	
 

	
 
/** Creates directories recursively */
 
int distren_mkdir_recurse(char *dirname)
 
{
 
  size_t counter;
 
  char *nextdir;
 

	
 
  nextdir = strdup(dirname);
 
  for(counter = 0; nextdir[counter]; counter ++)
 
    {
 
      /** @TODO OS-portabalize the path-separators */
 
      if(nextdir[counter] == '/')
 
        {
 
          nextdir[counter] = '\0';
 
          mkdir(nextdir, S_IRWXU | S_IRGRP | S_IROTH);
 
          nextdir[counter] = '/';
 
        }
 
    }
 

	
 
  return 0;
 
}
 

	
 
/**
 
 @TODO: Use for constructing path to job data locally and/or on data.distren.org
 
 */
 
int job_build_path(char *filename, unsigned int jobnum)
 
{
 
  return 0;
 
}
 

	
 

	
 
int downloadTar(char *url, char *destinationPath){
 
  // Prepare to download the job tar if it isn't  present
 
  struct stat buffer;
 
  int fstatus = stat(destinationPath, &buffer);
 
  if(fstatus == -1)
 
    {
 
      // Download the Tar
 
      int ret = curlget(url, destinationPath);
 
      if(ret == 0){
 
        fprintf(stderr, "Job data retrieved successfully\n");
 
        // free(url); @FIXME: causes doublefree! Curl must free the url?
 
        return 0;
 
      }
 
      else if(ret == 1){
 
        fprintf(stderr, "Downloading job data from %s failed. Check your network connection.\n",url);
 
        free(url);
 
        return 1; // Eventually make a retry loop
 
      }
 
    }
 
  else{
 
    fprintf(stderr, "Tar already exists! Download cancelled.\n");
 
    return 2;
 
  }
 

	
 
}
 

	
 

	
 
int uploadOutput(char *pathtoOutput, char *urltoOutput, int jobnum, int framenum, int slavekey){
 
  //fprintf(stderr,"Uploading output %s to url %s for job %d and frame %d for slavekey %d", pathtoOutput, urltoOutput, jobnum, framenum, slavekey);
 
  if( !curlpost(pathtoOutput, urltoOutput, jobnum, framenum, slavekey)) // Uploads output
 
    {
 
      fprintf(stderr,"Upload successful, removing old output...\n");
 
      remove(pathtoOutput); // Delete the file after its uploaded
 
      return 0;
 
    }
 
  else
 
    {
 
      fprintf(stderr,"Upload failed. Check your network connection. Retrying upload...\n");
 
      int tries=1;
 
      while(tries<=10 && curlpost(pathtoOutput, urltoOutput, jobnum, framenum, slavekey))
 
        {
 
          fprintf(stderr, "Upload failed. Trying again in 10 seconds... (attempt %d of 10)\n", tries);
 
          tries++;
 
          sleep(10);
 
        }
 
      return 1; // Upload failed after multiple tries
 
      // @FUTURE: Keep track of files that we were unable to upload, and upload them later
 
    }
 
}
 

	
 
/**
 
   Extracts archive to the specified directory, creating this directory
 
   if necessary (but this directory will not be recursively created).
 
   @param outdir output directory
 
   @param pathtoTar filename of the archive to extract
 
 */
 
int unpackJob(char *outdir, char *pathtoTar)
 
{
 
  int ret;
 

	
 
  struct archive *a;
 
  struct archive_entry *ae;
 
  int astatus;
 

	
 
  /* ignore return because directory may exist already */
 
  mkdir(outdir, 0700);
 
  ret = chdir(outdir);
 
  if(ret == -1)
 
    {
 
      perror("chdir");
 
      return 1;
 
    }
 

	
 
  a = archive_read_new();
 
  ae = archive_entry_new();
 

	
 
  archive_read_support_compression_all(a);
 
  archive_read_support_format_raw(a);
 
  astatus = archive_read_open_filename(a, pathtoTar, 8192);
 
  if (astatus != ARCHIVE_OK)
 
    {
 
      fprintf(stderr, "error opening archive\n");
 
      return 1;
 
    }
 

	
 
  for(astatus = ARCHIVE_OK;
 
      astatus == ARCHIVE_OK
 
	|| astatus == ARCHIVE_WARN;
 
      )
 
    {
 
      astatus = archive_read_next_header2(a, ae);
 
      if(astatus == ARCHIVE_WARN)
 
	fprintf(stderr, "Encountered nonfatal read error somehow\n");
 

	
 
      if(astatus == ARCHIVE_OK
 
	 || astatus == ARCHIVE_WARN)
 
	astatus = archive_read_extract(a, ae,
 
				       ARCHIVE_EXTRACT_NO_OVERWRITE
 
				       | ARCHIVE_EXTRACT_SECURE_SYMLINKS
 
				       | ARCHIVE_EXTRACT_SECURE_NODOTDOT);
 
    }
 
  archive_entry_free(ae);
 
  archive_read_finish(a);
 
  
 
  if(astatus != ARCHIVE_EOF)
 
    {
 
      fprintf(stderr, "Error reading archive\n");
 
      return 1;
 
    }
 

	
 
  return 0;
 
}
 

	
 

	
 
void prepareJobPaths(int jobnum, int framenum, char *outputExt, char *datadir, char **urltoTar,char **pathtoTar,char **pathtoTardir,char **pathtoJob, char **pathtoJobfile,char **urltoOutput,char **pathtoOutput, char **pathtoOutdir)
 
void prepareJobPaths(int jobnum, int framenum, char *outputExt, char *datadir, char **urltoTar,char **pathtoTar,char **pathtoTardir,char **pathtoJob, char **pathtoJobfile, char **urltoJobfile, char **urltoOutput,char **pathtoOutput, char **pathtoRenderOutput, char **pathtoOutdir)
 
{
 
  // Variable Preparation
 
  char *jobdatapath;
 
   _distren_asprintf(&jobdatapath, "job%d", jobnum);
 
   _distren_asprintf(urltoTar, "http://data.distren.org/job%d/job%d.tar.gz", jobnum, jobnum); // Prepares URL to download from
 
   _distren_asprintf(urltoJobfile, "http://data.distren.org/job%d/job.blend", jobnum); // Prepares URL to download from
 

	
 
   _distren_asprintf(pathtoTar, "%s/%s/job%d.tar.gz", datadir, jobdatapath, jobnum); // Prepares destination to save to
 

	
 
   _distren_asprintf(pathtoTardir, "%s/%s/", datadir, jobdatapath); // Prepares destination to save to
 

	
 
   _distren_asprintf(pathtoJob, "%s/%s/", datadir, jobdatapath);
 

	
 
   _distren_asprintf(pathtoJobfile, "%s/%s/job.blend", datadir, jobdatapath ); // Prepares the path to the jobfile
 
   _distren_asprintf(urltoOutput, "http://distren.org/slaveUpload.php"); // Prepares the URL where output is posted
 
   _distren_asprintf(pathtoOutput, "%s/%s/output/job%d-frame%d.%s", datadir, jobdatapath, jobnum, framenum, outputExt );
 
   _distren_asprintf(urltoOutput, "http://dev.distren.org/slaveUpload.php?jobkey=%d&framekey=%d",jobnum,framenum); // Prepares the URL where output is posted
 

	
 
   _distren_asprintf(pathtoRenderOutput, "%s/%s/output/job%d-frame#.%s", datadir, jobdatapath, jobnum, outputExt ); // Note: the # is for blender to put in framenum in output file
 
   _distren_asprintf(pathtoOutput, "%s/%s/output/job%d-frame%d.%s", datadir, jobdatapath, jobnum, framenum, outputExt ); // Note: the # is for blender to put in framenum in output file
 

	
 
   _distren_asprintf(pathtoOutdir, "%s/%s/output", datadir, jobdatapath);
 
   free(jobdatapath);
 
}
 

	
 
int checkUsername(char *username)
 
{
 
  if(username == NULL || strcmp(username, "!username") == 0 )
 
    {
 
     fprintf(stderr, "\nPlease ensure that your username is present in distrenslave.conf\n");
 
     return 1;
 
   }
 
   else
 
     if( username != NULL || strcmp(username, "!username") != 0 )
 
       {
 
         // Log in the user
 
         if(login_user(username) == 1){
 
           fprintf(stderr, "You should now be logged into distren.\n");
 
           return 0;
 
         }
 
         else
 
           {
 
             fprintf(stderr, "Login failed, please check your username. If you have not registered, please do so on the DistRen website.\n");
 
             return 1;
 
           }
 
       }
 
     else
 
       {
 
         fprintf(stderr, "Please check your distrenslave.conf, it appears to be incorrectly formatted.\n");
 
         return 1;
 
       }
 
}
 

	
 
void slaveTest(char *datadir)
 
{
 
  int command;
 
  int test = 1;
 
  int jobnum = 0;
 
  int framenum = 0;
 
  int slavekey = 0;
 
  fprintf(stderr,"Hello!\n");
 
  char tmpString1[100] = "";
 
  char tmpString2[100] = "";
 
  char *uploadURL = "http://distren.org/slaveUpload.php";
 

	
 
  while(test == 1)
 
   {
 
     fprintf(stderr, "Welcome to DistRenSlave Alpha Interactive Test Mode\n\n");
 
     fprintf(stderr, "\t1 \tTest posting (uploading) data\n");
 
@@ -539,96 +555,118 @@ void slaveTest(char *datadir)
 
     fprintf(stderr, "\t4 \tTest archive decompression\n");
 
     fprintf(stderr, "\t5 \tQuit\n");
 

	
 
     scanf("%d", &command);
 

	
 
     switch(command)
 
     {
 
     case 1:
 
       fprintf(stderr,"Path to output file to upload: ");
 
       scanf("%99s", tmpString1);
 
       fprintf(stderr,"Framenum that is being uploaded: ");
 
       scanf("%d", &framenum);
 
       fprintf(stderr,"Jobnum that frame belongs to: ");
 
       scanf("%d", &jobnum);
 
       fprintf(stderr,"Slavekey that rendered this output: ");
 
       scanf("%d", &slavekey);
 
       uploadOutput(tmpString1, uploadURL, jobnum, framenum, slavekey);
 
       break;
 
     case 2:
 
       fprintf(stderr,"Remotio not implemented yet!\n");
 
       break;
 
     case 3:
 
       fprintf(stderr,"URL to file: ");
 
       scanf("%s", tmpString1);
 
       fprintf(stderr,"Local save path (including filename): ");
 
       scanf("%s", tmpString2);
 
       if(downloadTar(tmpString1, tmpString2))
 
         fprintf(stderr,"Error downloading!\n");
 
       else
 
         fprintf(stderr,"Success!\n");
 
       break;
 
     case 4:
 
       fprintf(stderr,"Jobnum to decompress: ");
 
       scanf("%d", &jobnum);
 
       fprintf(stderr,"Path to compressed data: ");
 
       scanf("%s", tmpString2);
 
       unpackJob(datadir, tmpString1);
 
       break;
 
     case 5:
 
       test = 0;
 
       break;
 
     default:
 
       fprintf(stderr, "Invalid input, please try again.\n");
 
       break;
 
     }
 
   }
 
}
 

	
 
/*
 
int runBenchmark(int slavekey, char *datadir){
 
  // Execute blender
 

	
 
  if(exec_blender(pathtoJobfile, pathtoOutput, framenum))
 
    {
 
      fprintf(stderr,"Error running Blender. Check your installation and/or your PATH.\n");
 
      _web_resetframe(slavekey, password, jobnum, framenum);  // Unassign the frame on the server so other slaves can render it
 
      return 1;
 
    }
 
  free(pathtoJobfile);
 

	
 
  struct stat buffer;
 
  int fstatus = stat(pathtoOutput, &buffer);
 
  if(fstatus == -1){
 
    fprintf(stderr,"*** Frame was not rendered correctly! Scene may not have camera, or your blender installation is not working.\n");
 
    _web_resetframe(slavekey, password, jobnum, framenum);  // Unassign the frame on the server so other slaves can render it
 
    return 1;
 
  }
 

	
 
}
 
*/
 

	
 
/* simpleSlave functions */
 

	
 
/** Memory struct for curl */
 
struct _web_memorystruct {
 
  char *memory;
 
  size_t size;
 
};
 

	
 
static void *_web_myrealloc(void *ptr, size_t size);
 

	
 
static void *_web_myrealloc(void *ptr, size_t size)
 
{
 
  /* There might be a realloc() out there that doesn't like reallocing
 
     NULL pointers, so we take care of it here */
 
  if(ptr)
 
    return realloc(ptr, size);
 
  else
 
    return malloc(size);
 
}
 

	
 
static size_t _web_writememorycallback(void *ptr, size_t size, size_t nmemb, void *data)
 
{
 
  size_t realsize = size * nmemb;
 
  struct _web_memorystruct *mem = (struct _web_memorystruct *)data;
 

	
 
  mem->memory = _web_myrealloc(mem->memory, mem->size + realsize + 1);
 
  if (mem->memory) {
 
    memcpy(&(mem->memory[mem->size]), ptr, realsize);
 
    mem->size += realsize;
 
    mem->memory[mem->size] = 0;
 
  }
 
  return realsize;
 
}
 

	
 
struct _web_memorystruct _web_getrequest(char *url){
 
  // fprintf(stderr,"Preparing to get request at %s",url);
 

	
 
  CURL *curl;
 
  CURLcode res;
 
  struct _web_memorystruct chunk;
 

	
 
  chunk.memory=NULL; /* we expect realloc(NULL, size) to work */
 
  chunk.size = 0;    /* no data at this point */
 

	
 
  curl = curl_easy_init();
 
  if(curl) {
 
    curl_easy_setopt(curl, CURLOPT_URL, url);
src/server/slavefuncs.h
Show inline comments
 
@@ -4,65 +4,65 @@
 
  This file is a part of DistRen.
 

	
 
  DistRen is free software: you can redistribute it and/or modify
 
  it under the terms of the GNU Affero General Public License as published by
 
  the Free Software Foundation, either version 3 of the License, or
 
  (at your option) any later version.
 

	
 
  DistRen is distributed in the hope that it will be useful,
 
  but WITHOUT ANY WARRANTY; without even the implied warranty of
 
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
  GNU Affero General Public License for more details.
 

	
 
  You should have received a copy of the GNU Affero General Public License
 
  along with DistRen.  If not, see <http://www.gnu.org/licenses/>.
 
*/
 

	
 
#ifndef _DISTREN_SLAVEFUNCS_H
 
#define _DISTREN_SLAVEFUNCS_H
 

	
 
#include "distrenjob.h"
 

	
 
#include "common/remoteio.h"
 

	
 
#include <libxml/xpath.h>
 
#include <stdio.h>
 

	
 

	
 
struct msg;
 
int sendSignal(struct remoteio *rem, char signal);
 
int sendExtSignal(struct remoteio *rem, char signal, char *data);
 
xmlNodePtr xml_quickxpath(xmlXPathContextPtr xpathctxt, xmlChar *path);
 
int software_updatecheck();
 
int delete_jobdata(int jobnum, char *datadir);
 
size_t curl_writetodisk(void *ptr, size_t size, size_t nmemb, FILE *stream);
 
int curlget(char *url, char *out);
 
int curlpost(char *filename, char *url, int jobnum, int framenum, int slavekey);
 
int ssh_keygen();
 
int register_user(char *username, char *email);
 
int login_user(char *username);
 
int conf_replace(char *conffile, char *wordtoreplace, char *replacewith);
 
int exec_blender(char *input, char *output, int frame);
 
void xmlinit();
 
void xmlcleanup();
 
int distren_mkdir_recurse(char *dirname);
 
int job_build_path(char *filename, unsigned int jobnum);
 
int downloadTar(char *url, char *destinationPath);
 
int uploadOutput(char *pathtoOutput, char *urltoOutput, int jobnum, int framenum, int slavekey);
 
int unpackJob(char *outdir, char *pathtoTar);
 
void prepareJobPaths(int jobnum, int framenum, char *outputExt, char *datadir, char **urltoTar,char **pathtoTar,char **pathtoTardir,char **pathtoJob, char **pathtoJobfile,char **urltoOutput,char **pathtoOutput, char **pathtoOutdir);
 
void prepareJobPaths(int jobnum, int framenum, char *outputExt, char *datadir, char **urltoTar,char **pathtoTar,char **pathtoTardir,char **pathtoJob, char **pathtoJobfile,char **urltoJobfile,char **urltoOutput,char **pathtoOutput, char **pathtoRenderOutput, char **pathtoOutdir);
 
int checkUsername(char *username);
 
void slaveTest();
 

	
 
/* Simple slave */
 
struct _web_memorystruct;
 
static void *_web_myrealloc(void *ptr, size_t size);
 
static size_t _web_writememorycallback(void *ptr, size_t size, size_t nmemb, void *data);
 
struct _web_memorystruct _web_getrequest(char *url);
 
void _web_finishframe(int slavekey, char *slavepass, int jobnum, int framenum);
 
void _web_startframe(int slavekey, char *slavepass, int jobnum, int framenum);
 
void _web_resetframe(int slavekey, char *slavepass, int jobnum, int framenum);
 
int _web_getwork(int slavekey, char *slavepass, int *jobnum, int *framenum);
 
void _web_setrenderpower(int slavekey, char *slavepass, int jobnum, int framenum);
 

	
 

	
 
#endif
0 comments (0 inline, 0 general)