Changeset - b236a45982d4
[Not reviewed]
default
0 3 0
Nathan Brink (binki) - 16 years ago 2009-09-13 16:20:45
ohnobinki@ohnopublishing.net
fixed delete_jobdata() slightly
3 files changed with 7 insertions and 5 deletions:
0 comments (0 inline, 0 general)
src/server/slave.c
Show inline comments
 
/*
 
  Copyright 2009 Nathan Phillip Brink, Ethan Zonca, Matthew Orlando
 

	
 
  This file is a part of DistRen.
 

	
 
  DistRen is free software: you can redistribute it and/or modify
 
  it under the terms of the GNU Affero General Public License as published by
 
  the Free Software Foundation, either version 3 of the License, or
 
  (at your option) any later version.
 

	
 
  DistRen is distributed in the hope that it will be useful,
 
  but WITHOUT ANY WARRANTY; without even the implied warranty of
 
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
  GNU Affero General Public License for more details.
 

	
 
  You should have received a copy of the GNU Affero General Public License
 
  along with DistRen.  If not, see <http://www.gnu.org/licenses/>.
 

	
 
 */
 

	
 
// Provides DISTREN_REQUEST* which is in reality, DISTREN_SEND(signal) in the minds of your average joe
 
#include "protocol.h"
 
#include "options.h" // Confuse, etc.
 
#include "slavefuncs.h" // Slave functions
 
#include "asprintf.h"
 
#include "remoteio.h"
 

	
 
#include <string.h>
 
#include <stdio.h>
 
#include <unistd.h>
 
#include <sys/stat.h> /*< mkdir() */
 
#include <stdlib.h>
 

	
 
int slavestatus = 0; // Ugh global vars
 

	
 
int main(int argc, char *argv[])
 
{
 

	
 
int counter;
 

	
 
/* Parses arguments, skips if there are no args */
 
for(counter=0; counter<argc; counter++){
 
	if(strcmp(argv[counter], "-h") == 0) {
 
		fprintf(stderr, "Usage: distrenslave [option] <username> <emailaddr>\nStarts a distren slave\n\t-h\tshow this help\n\t-c\tregisters a user with [username] and [emailaddr] \n");
 
		return 2;
 
	}
 
}
 
	if(argc>1
 
		&& (strcmp("-c", argv[1]) == 0))
 
	{
 
		char *username;
 
		char *email;
 

	
 
		if(argc != 4)
 
		{
 
			fprintf(stderr, "I need more arguments!\n%d is not enough!\n Invoke 'distrenslave -c <username> <emailaddr> to register. \n Already registered? Edit your distrenslave.conf file!\n'", argc - 1);
 
			return 234;
 
		}
 

	
 
		username = argv[2];
 
		email = argv[3];
 

	
 
		if(!strchr(email, '@'))
 
		{
 
			fprintf(stderr, "I want to see an '@' in your email address!\n%s may be good enough for you, but I need more!\n", email);
 
			return 235;
 
		}
 
		if( register_user(username, email) == 1){
 
			// register_user generates a key, too */
 
			fprintf(stderr,"Registration successfull. You may now invoke distrenslave with no arguments.\n");
 
			return 0;
 
		}
 
		else{
 
			fprintf(stderr,"Please try again! :D\n");
 
			return 0;
 
		}
 
	}
 
/* End arg parser */
 

	
 
/* Option getter: Creates vars to grab stuff from conf, uses the options include to grab this data */
 
char *username;
 
char *datadir;
 
cfg_t * my_cfg;
 
cfg_opt_t myopts[] = {
 
		CFG_SIMPLE_STR("username", &username),
 
		CFG_SIMPLE_STR("datadir", &datadir),
 
		CFG_END()
 
		};
 
struct options_common *commonopts;
 

	
 
username = NULL;
 
options_init(argc,argv,&my_cfg, myopts, "slave", &commonopts);
 
/* End option getter */
 

	
 

	
 
/* Notifies the user if there is no username in the conf file */
 
	if(username == NULL || strcmp(username, "!username") == 0 ){
 
	  fprintf(stderr, "\nYou didn't register!\nPlease register or edit your config. (see -h)\nIf this error persists, check distrenslave.conf to ensure all items are filled.\n");
 
	}
 
	else if( username != NULL || strcmp(username,"!username") != 0 ){
 
		// Logs ya in:
 
		if(login_user(username) == 1){
 
			fprintf(stderr,"You should now be logged into distren.\n");
 
		}
 
		else{
 
			fprintf(stderr,"Login failed. I have no clue why. Sorry.\n");
 
			return 0;
 
		}
 
	}
 
	else{
 
	  fprintf(stderr,"Something is terribly wrong!");
 
	}
 

	
 

	
 

	
 
/* Code-filled Variables  */
 
int jobnum;          // The job number that we're currently working
 
int framenum;        // @TODO: Remotio should fill this
 

	
 
int gotframe;        // set this to 1 after data for a job is received from the server
 
int busy = 0;        // Client business 1=busy 0=idle
 

	
 
char *urltoTar;      // Full URL to the server-side location of job#.tgz
 
char *pathtoTar;     // Full path to the location of the job#.tgz
 

	
 
char *urltoOutput;   // Full URL where output is posted
 
char *pathtoJobfile; // Full path to the job's main file
 
char *pathtoXml;     // Full path to the job's xml file
 
char *pathtoOutput;  // Full path to the output (rendered) file
 
char *outputExt;     // Output Extension (e.g., JPG)
 

	
 
int threads = sysconf(_SC_NPROCESSORS_ONLN); // Equals the number of currently online processors
 
// FOR Win32: int threads = sysinfo.dwNumberOfProcessors;
 

	
 
struct distrenjob *myjob; // Structure to hold data gathered from the XML file
 

	
 
/* If the server says that every frame for jobnum is finished */
 
if( 1 == 0){
 
delete_jobdata(jobnum);
 
if( 1 == 0)
 
  {
 
    delete_jobdata(jobnum, datadir);
 
}
 

	
 
// If the slave is getting job info...
 
if(gotframe ==1)
 
  {
 
    /* @TODO: fix these remoteio_read's */
 
    jobnum = remoteio_read(jobnum); // Set jobnum from remoteio (we could use info from struct, but we need this info to download the xmlfile)
 
    framenum = remoteio_read(jobnum); // Set framenum from remoteio
 

	
 
    char *tarcmd;
 
    char *outdir;
 
    char *jobdatapath;
 

	
 
    fprintf(stderr, "Received %d in job %d, preparing to render...\n",framenum,jobnum);
 

	
 
    /**
 
       Variable Preparation
 
       @todo find where to free() all of these
 
    */
 
    _distren_asprintf(&jobdatapath, "job%d", jobnum);
 
    _distren_asprintf(&urltoTar, "http://protofusion.org/distren/stor/job%d/job%d.tar.gz",jobnum); // Prepares URL to download from
 
    _distren_asprintf(&pathtoTar, "%s/stor/jobdata/job%d.tar.gz", datadir, jobnum); // Prepares destination to save to
 

	
 
    _distren_asprintf(&pathtoJobfile, "%s/%s/job.blend", datadir, jobdatapath ); // Prepares the path to the jobfile
 
    _distren_asprintf(&urltoOutput, "http://protofusion.org/distren/stor/job%d/output/", jobdatapath ); // Prepares the URL where output is posted
 
    _distren_asprintf(&pathtoXml, "%s/job%d/job%d.xml",datadir, jobnum ); // Prepares the path to the job's XML file
 

	
 
      // Downloads the job tar if it isn't  present. @TODO: Delete old job data
 
      struct stat buffer;
 
      int fstatus = stat(pathtoJobfile, &buffer);
 

	
 
      // If stat'ing the jobfile shows that the jobfile doesn't exist, we'll download it.
 
      if(fstatus == -1){
 
              // Downloads the Tar @TODO: add progress bar
 
              if( curlget(urltoTar, pathtoTar) == 0){
 
                fprintf(stderr, "File downloaded without errors\n");
 
              }
 
              else{
 
                fprintf(stderr, "Download tar from server failed. Either the server is down, or the job hosting system is screwed up.\nContact admin@protofusion.org and include this message.\nGoodbye. Better luck next time.\n");
 
              }
 
      }
 
      else{
 
        fprintf(stderr, "Using cached job file...\n");
 
      }
 

	
 
    _distren_asprintf(&outdir, "/tmp/distren/job%d", jobnum); /*< @todo free() */
 
    mkdir("/tmp/distren", 0750); /* @TODO: Make this less *nix-specific */
 
    mkdir(outdir, 0750);
 

	
 
    _distren_asprintf(&tarcmd, "tar -xvf \"%s\" -C \"%s\"", pathtoTar, outdir); /* @TODO: Make this portable. Libtar or something? */
 
    system(tarcmd);
 
    free(tarcmd);
 

	
 
    // Parses a job's XML file, puts data in the myjob struct
 
    if(xml2distrenjob(&myjob, pathtoXml) == 0){
 
      fprintf(stderr,"Well, the XML craziness may have worked. Maybe. \n");
 
      distrenjob_free(&myjob); // Frees things up if it was successful. xml2distrenjob() really (usually) only fails if malloc'ing inside it fails
 
    }
 
    else{
 
      fprintf(stderr,"I think the XML craziness may have failed, so I'll terminate just for fun.\n");
 
      return 1;
 
    }
 

	
 
    /* Variable-fillers which require XML */
 
    outputExt = myjob->output_format; /* @TODO: FIXME! */
 
    _distren_asprintf(&pathtoOutput, "%s/job%d/output/frame%d.%s", datadir, jobnum, framenum, outputExt ); // Prepares the path to the jobfile
 

	
 

	
 
    /** Series of IF's for jobtype based on xml content */
 
    if(1==1){
 
     exec_blender(myjob, pathtoJobfile, pathtoOutput, framenum, threads); // @TODO: This warning should be fixed :D
 
    }
 

	
 
    // Consider placing the following in the exec_blender() function
 
    while(busy == 1){
 
      tell_the_server(DISTREN_REQUEST_PROGRESS);
 
        fprintf(stderr, "Rendering frame %d in job %d...",framenum,jobnum);
 
        sleep(5); // or not... this should be more event-driven, but should still give a heartbeat to the server
 
      }
 
    busy = 2;
 
    // busy = 1; // @TODO: When done rendering, set this.
 
     }
 
    // When blender is finished, run this...
 
      if(busy==2){
 
        fprintf(stderr, "Finished frame %d in job %d, uploading...",framenum,jobnum);
 
        char *outputurl;
 
        _distren_asprintf(&outputurl, "http://protofusion.org/distren/stor/job%d/",jobnum); // Aggregates the output url
 
        curlpost(pathtoOutput, urltoOutput); // uploads (HTML POST) the output at outputpath to the server at outputurl
 
        tell_the_server(DISTREN_REQUEST_DONEFRAME); // Tells the server that it's finished rendering this frame
 
        busy=0;
 
        // Slave now becomes idle, doesn't need to tell the server anything, ssh handles this.
 
      }
 

	
 
  return 0;
 
}
src/server/slavefuncs.c
Show inline comments
 
/*
 
  Copyright 2009 Nathan Phillip Brink, Ethan Zonca, Matthew Orlando
 

	
 
  This file is a part of DistRen.
 

	
 
  DistRen is free software: you can redistribute it and/or modify
 
  it under the terms of the GNU Affero General Public License as published by
 
  the Free Software Foundation, either version 3 of the License, or
 
  (at your option) any later version.
 

	
 
  DistRen is distributed in the hope that it will be useful,
 
  but WITHOUT ANY WARRANTY; without even the implied warranty of
 
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
  GNU Affero General Public License for more details.
 

	
 
  You should have received a copy of the GNU Affero General Public License
 
  along with DistRen.  If not, see <http://www.gnu.org/licenses/>.
 
*/
 

	
 
 /*
 
  * Registration on server. Needs attention. Prevent account spamming.
 
  * distrenslave -c username email@example.com
 
 */
 

	
 
#include "asprintf.h"
 
#include "slavefuncs.h"
 
#include "distrenjob.h"
 
#include "execio.h"
 

	
 
#include <curl/curl.h>
 
#include <curl/types.h>
 
#include <curl/easy.h>
 

	
 
#include <stdio.h>
 
#include <string.h>
 
#include <unistd.h>
 
#include <stdlib.h>
 
#include <sys/stat.h>
 
#include <fcntl.h>
 

	
 

	
 
/**
 
 utility function for XPath-ish stuff:
 
 */
 
xmlNodePtr xml_quickxpath(xmlXPathContextPtr xpathctxt, xmlChar *path)
 
{
 
  xmlNodePtr toreturn;
 

	
 
  xmlXPathObjectPtr xmlxpathobjptr;
 
  xmlxpathobjptr = xmlXPathEval(path, xpathctxt);
 
  if(!xmlxpathobjptr
 
     || !xmlxpathobjptr->nodesetval->nodeNr)
 
    {
 
      fprintf(stderr, "XPath resolution failed for ``%s'' in ``%s'' (``%s'')\n", path, xpathctxt->doc->name, xpathctxt->doc->URL);
 
      return (xmlNodePtr)NULL;
 
    }
 

	
 
  toreturn = *(xmlxpathobjptr->nodesetval->nodeTab);
 

	
 
  xmlXPathFreeObject(xmlxpathobjptr);
 

	
 
  return toreturn;
 
}
 

	
 

	
 
/** Stub for deleting job data from the disk. @TODO: unstubify me! */
 
int delete_jobdata(int jobnum, char datadir){
 
int delete_jobdata(int jobnum, char *datadir)
 
{
 
  char *jobpath;
 
  _distren_asprintf(&jobpath, "%s/%d",datadir,jobnum);
 
  rmdir(jobpath);
 
  fprintf(stderr,"I just failed to remove all of your job data because I can only delete empty directories! Haha!\nPlease contact the dev team :D\n");
 
  fprintf(stderr, "I just failed to remove all of your job data because I can only delete empty directories! Haha!\nPlease contact the dev team :D\n\tWe'd be overjoyed to know that someone was willing to try to use this in this devel/design/planning stage ;-)\n");
 
  return 0;
 
}
 

	
 
/** Stub stub stubbiness ugh @TODO: Kill me. */
 
void tell_the_server(char *stuff){
 
}
 

	
 
/** Function referenced by curlget() to write data to disk. */
 
size_t curl_writetodisk(void *ptr, size_t size, size_t nmemb, FILE *stream)
 
 {
 
    return fwrite(ptr, size, nmemb, stream);
 
  }
 

	
 
/** Helper function for cURL's progressbar */
 
int curl_progress( char *Bar,double t,double d,double ultotal,double ulnow)
 
{
 
fprintf(stderr,"Downloading... %f%% complete\r",d/t*100);
 
return 0;
 
}
 

	
 
/** Retrieves a URL with cURL and saves it to disk */
 
int curlget(char *url, char *out){
 

	
 
  double *Bar; // Stores cURL progressbar info
 
  CURL *curl;
 
  CURLcode res;
 
  FILE *outfile;
 

	
 
  curl = curl_easy_init();
 
  if(curl) {
 
	outfile = fopen(out, "w"); // Open where we're writing to
 

	
 
  curl_easy_setopt(curl, CURLOPT_URL, url);
 
  curl_easy_setopt(curl, CURLOPT_WRITEDATA, outfile);
 
  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_writetodisk); // this MUST be set for win32 compat.
 
  curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
 
  curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, curl_progress);
 
  curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &Bar);
 
  res = curl_easy_perform(curl);
 
  curl_easy_cleanup(curl);
 
  }
 
  fprintf(stderr,"\n"); // Clears out he progressbar's carriage return
 
  return res; // 0 means OK, nonzero means AAAH badness
 
}
 

	
 
/** Posts a file to a url with cUrl */
 
int curlpost(char *filename, char *url){
 
  char *targetname = "uploadedfile"; // Name of the target in the php file on the server (Don't change me unless you have different PHP code)
 
  CURL *curl;
 
  CURLcode res;
 
  struct curl_httppost *formpost=NULL;
 
  struct curl_httppost *lastptr=NULL;
 
  struct curl_slist *headerlist=NULL;
 
  static const char buf[] = "Expect:";
 

	
 
  curl_global_init(CURL_GLOBAL_ALL);
 

	
 
  /* upload field... */
 
  curl_formadd(&formpost,
 
               &lastptr,
 
               CURLFORM_COPYNAME, targetname,
 
               CURLFORM_FILE, filename,
 
               CURLFORM_END);
 
  /* filename field... */
 
  curl_formadd(&formpost,
 
               &lastptr,
 
               CURLFORM_COPYNAME, "filename",
 
               CURLFORM_COPYCONTENTS, filename,
 
               CURLFORM_END);
 
  /* submit field, not usually needed, just in case... */
 
  curl_formadd(&formpost,
 
               &lastptr,
 
               CURLFORM_COPYNAME, "submit",
 
               CURLFORM_COPYCONTENTS, "send",
 
               CURLFORM_END);
 

	
 
  curl = curl_easy_init();
 
  headerlist = curl_slist_append(headerlist, buf);
 
  if(curl) {
 
    /* Setting the URL to get the post, and the contents of the post */
 
    curl_easy_setopt(curl, CURLOPT_URL, url);
 
    curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
 
    res = curl_easy_perform(curl);
 

	
 
    curl_easy_cleanup(curl);
 
    /* cleanup the formpost junk */
 
    curl_formfree(formpost);
 
    curl_slist_free_all (headerlist);
 
  }
 
  return 0;
 
}
 

	
 
/** Generates a SSH key with ssh-keygen */
 
int ssh_keygen(){
 
	/* Checks to see if the keys are already present. */
 
	int status;
 
	struct stat buffer;
 
	status = stat(SYSCONFDIR "/distren.id_rsa", &buffer);
 
	if(status != -1){
 
		fprintf(stderr, "***Please delete etc/distren.id_rsa and etc/distren.id_rsa.pub to register.\n");
 
		return 0;
 
	}
 

	
 
	/* start execio code */
 
	char *command = "ssh-keygen"; // @TODO: append .exe if win32?
 
	int ret;
 
	char *cmd[] = { command, "-q", "-f", SYSCONFDIR "/distren.id_rsa", "-N", "", (char *)NULL }; // TODO: Give me the correct args!
 
	char buf[10];
 
	struct execio *testrem;
 
	size_t readlen;
 
	ret = execio_open(&testrem, command, cmd); // @TODO: This path will be absolute for testing, should be relative to install on production
 
	buf[9] = '\0';
 
	while(!execio_read(testrem, buf, 9, &readlen))
 
	    {
 
	      if(readlen > 9) {
 
		fprintf(stderr, "Something is terribly wrong!\n");
 
	      }
 
	      buf[readlen] = '\0';
 
	      fprintf(stderr, "read \"%s\"\n", buf);
 
	    }
 
	execio_close(testrem);
 
	/* end execio code */
 

	
 
	// Supposedly execio returns 1 if it has bad args.
 
	if(ret == 1){
 
		fprintf(stderr, "Generating your key failed. Ensure that ssh-keygen is present!\n"); // Use different executor that searches the path? there is one...
 
		return 0;
 
	}
 
	else{
 
		fprintf(stderr,"We successfully generated your key! Yay!\n");
 
		return 1;
 
	}
 
return 0;
 
}
 

	
 
/** Registers the user on the DistRen server */
 
int register_user(char *username, char *email)
 
{
 

	
 
/* Note: this code moved here from after the useradd code, so useradd doesn't happen if there is an existing key, etc */
 
  /* puts the person's username in the conf */
 
  if(conf_replace(username) == 0){
 
	fprintf(stderr, "Failed!\n");
 
	return 0;
 
	}
 

	
 
  /* generates keys for login, @TODO: pub key must somehow be sent to the server. */
 
  if(ssh_keygen() == 0){
 
	  fprintf(stderr, "Failed!\n");
 
	  return 0;
 
  }
 

	
 
  /*
 
   * All created user accounts should be sandboxed accordingly, requiring a different skel, and the default shell to be rbash. Also,
 
   * a custom path defined in the .bashrc of the skel is needed.
 
   */
 
  int ret;
 
  char buf[10];
 
  struct execio *testrem;
 
  char *execargv[] =
 
    {
 
      "ssh",
 
      "distren_setup@protofusion.org",
 
      "-i",
 
      SYSCONFDIR "/setup_rsa", // @TODO: How will we distribute this key?
 
      "-p",
 
      "23",
 
      "newuser",
 
      "-M",
 
      "-c",
 
      email,
 
      "-d",
 
      "/home/distren",
 
      "--gid",
 
      "541",
 
      username,
 
      (char *)NULL
 
    };
 
  size_t readlen;
 
 ret = execio_open(&testrem, "ssh", execargv); // TODO: Grab returns from this someday.
 
  buf[9] = '\0';
 
  while(!execio_read(testrem, buf, 9, &readlen))
 
    {
 
      if(readlen > 9) {
 
	fprintf(stderr, "!!!! Something is terribly wrong!\n");
 
      }
 
      buf[readlen] = '\0';
 
      fprintf(stderr, "read \"%s\"\n", buf);
 
    }
 
  execio_close(testrem);
 

	
 
  /* @TODO: Parse the output buffer or something to check when user creation fails due to duplicate users. This is pretty important. */
src/server/slavefuncs.h
Show inline comments
 
/*
 
  Copyright 2009 Nathan Phillip Brink, Ethan Zonca, Matthew Orlando
 

	
 
  This file is a part of DistRen.
 

	
 
  DistRen is free software: you can redistribute it and/or modify
 
  it under the terms of the GNU Affero General Public License as published by
 
  the Free Software Foundation, either version 3 of the License, or
 
  (at your option) any later version.
 

	
 
  DistRen is distributed in the hope that it will be useful,
 
  but WITHOUT ANY WARRANTY; without even the implied warranty of
 
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
  GNU Affero General Public License for more details.
 

	
 
  You should have received a copy of the GNU Affero General Public License
 
  along with DistRen.  If not, see <http://www.gnu.org/licenses/>.
 
*/
 

	
 
#ifndef _DISTREN_SLAVEFUNCS_H
 
#define _DISTREN_SLAVEFUNCS_H
 

	
 
#include "distrenjob.h"
 

	
 
#include <libxml/xpath.h>
 
#include <stdio.h>
 

	
 
xmlNodePtr xml_quickxpath(xmlXPathContextPtr xpathctxt, xmlChar *path);
 
void tell_the_server(char *stuff);
 
int delete_jobdata(int jobnum, char datadir);
 
int delete_jobdata(int jobnum, char *datadir);
 
size_t curl_writetodisk(void *ptr, size_t size, size_t nmemb, FILE *stream);
 
int curlget(char *url, char *out);
 
int curlpost(char *filename, char *url);
 
int ssh_keygen();
 
int register_user(char *username, char *email);
 
int login_user(char *username);
 
int conf_replace(char *username);
 
void exec_blender(struct distrenjob* distrenjob, char *input, char *output, int frame, int threads);
 
void xmlinit();
 
void xmlcleanup();
 

	
 
#endif
0 comments (0 inline, 0 general)