/*
Copyright 2009 Nathan Phillip Brink, Ethan Zonca, Matthew Orlando
This file is a part of DistRen.
DistRen is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
DistRen is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with DistRen. If not, see .
*/
#include "distrenjob.h"
#include "slavefuncs.h"
#include "common/asprintf.h"
#include "common/execio.h"
#include "common/protocol.h"
#include "common/remoteio.h"
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
/**
Sends the server a single request (see protocol.h)
*/
int sendSignal(struct remoteio *rem, char signal)
{
size_t written;
size_t towrite;
char *ssignal;
_distren_asprintf(&ssignal, "%c", signal);
towrite = strlen(ssignal);
while( towrite
&& !remoteio_write(rem, ssignal, towrite, &written))
{
fprintf(stderr, "Sending request...\n");
towrite -= written;
}
if(written)
return 0;
/**
if remoteio_write returned 1, the connection
is probably dead or there was a real error
*/
return 1;
}
/**
Sends the server an extended signal (request + data)
*/
int sendExtSignal(struct remoteio *rem, char signal, char *data){
size_t written;
size_t towrite;
char *ssignal;
_distren_asprintf(&ssignal, "%c%s", signal, data); // Just append the data FIXME: We should do this differently
towrite = strlen(ssignal);
while( towrite
&& !remoteio_write(rem, ssignal, towrite, &written))
{
fprintf(stderr, "Sending request...\n");
towrite -= written;
}
if(written)
return 0;
/**
if remoteio_write returned 1, the connection
is probably dead or there was a real error
*/
return 1;
}
/**
Grabs the xml DOM node reached by an XPath.
@param path an XPath that lead to DOM node
@return the first node associated with the path or NULL if there is no match
*/
xmlNodePtr xml_quickxpath(xmlXPathContextPtr xpathctxt, xmlChar *path)
{
xmlNodePtr toreturn;
xmlXPathObjectPtr xmlxpathobjptr;
xmlxpathobjptr = xmlXPathEval(path, xpathctxt);
if(!xmlxpathobjptr
|| !xmlxpathobjptr->nodesetval->nodeNr)
{
fprintf(stderr, "XPath resolution failed for ``%s'' in ``%s'' (``%s'')\n", path, xpathctxt->doc->name, xpathctxt->doc->URL);
return (xmlNodePtr)NULL;
}
toreturn = *(xmlxpathobjptr->nodesetval->nodeTab);
xmlXPathFreeObject(xmlxpathobjptr);
return toreturn;
}
/** Stub for deleting job data from the disk. @TODO: unstubify me! */
int delete_jobdata(int jobnum, char *datadir)
{
char *jobpath;
_distren_asprintf(&jobpath, "%s/%d", datadir, jobnum);
// rmdir(jobpath);
fprintf(stderr, "Please manually remove %s. Automatic removal is currently not implemented.\n", jobpath);
return 0;
}
/** Function referenced by curlget() to write data to disk. */
size_t curl_writetodisk(void *ptr, size_t size, size_t nmemb, FILE *stream)
{
return fwrite(ptr, size, nmemb, stream);
}
/** Helper function for cURL's progress display */
int curl_progress( char *Bar,double t,double d,double ultotal,double ulnow)
{
fprintf(stderr,"Downloading: %f%% complete\r",d/t*100);
return 0;
}
/** Retrieves a URL with cURL and saves it to disk */
int curlget(char *url, char *out){
fprintf(stderr,"Preparing to download %s",url);
double *Bar; // Stores cURL progress display info
CURL *curl;
CURLcode res;
FILE *outfile;
curl = curl_easy_init();
if(curl) {
outfile = fopen(out, "w"); // Open where we're writing to
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, outfile);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_writetodisk); // this MUST be set for win32 compat.
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, curl_progress);
curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &Bar);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
fprintf(stderr,"\n"); // Clears out he progressbar's carriage return
return res; // 0 is OK, 1 is 404 or other error
}
/** Posts a file to a url with cUrl */
int curlpost(char *filename, char *url){
char *targetname = "uploadedfile"; // Name of the target in the php file on the server (Don't change me unless you have different PHP code)
CURL *curl;
CURLcode res;
struct curl_httppost *formpost=NULL;
struct curl_httppost *lastptr=NULL;
struct curl_slist *headerlist=NULL;
static const char buf[] = "Expect:";
curl_global_init(CURL_GLOBAL_ALL);
/* upload field... */
curl_formadd(&formpost,
&lastptr,
CURLFORM_COPYNAME, targetname,
CURLFORM_FILE, filename,
CURLFORM_END);
/* filename field... */
curl_formadd(&formpost,
&lastptr,
CURLFORM_COPYNAME, "filename",
CURLFORM_COPYCONTENTS, filename,
CURLFORM_END);
/* submit field, not usually needed, just in case... */
curl_formadd(&formpost,
&lastptr,
CURLFORM_COPYNAME, "submit",
CURLFORM_COPYCONTENTS, "send",
CURLFORM_END);
curl = curl_easy_init();
headerlist = curl_slist_append(headerlist, buf);
if(curl) {
/* Setting the URL to get the post, and the contents of the post */
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
/* cleanup the formpost junk */
curl_formfree(formpost);
curl_slist_free_all (headerlist);
}
return res;
}
/** Logs the user into the server after ensuring that keys exist */
int login_user(char *username)
{
// @TODO: Put some telnet-style auth code here unless this is obselete
return 1; // success
}
/** Replaces wordtoreplace with replacewith in conffile (relative to SYSCONFDIR) */
int conf_replace(char *conffile, char *wordtoreplace, char *replacewith){
int maxlinelen = 120;
char *fileOrig;
char *fileRepl;
_distren_asprintf(&fileOrig, "%s/%s", SYSCONFDIR, conffile);
_distren_asprintf(&fileRepl, "%s%s.edited", SYSCONFDIR, conffile);
char buffer[maxlinelen+2];
char *buff_ptr, *find_ptr;
FILE *fp1, *fp2;
size_t find_len = strlen(wordtoreplace);
fp1 = fopen(fileOrig,"r");
fp2 = fopen(fileRepl,"w");
if (fp1 ==NULL){
fprintf(stderr, "%s doesn't exist\n",fileOrig);
return 0;
}
else if(fp2 ==NULL){
fprintf(stderr, "Can't write a file to disk! Check permissions.\n");
return 0;
}
else{
while(fgets(buffer,maxlinelen+2,fp1))
{
buff_ptr = buffer;
while ((find_ptr = strstr(buff_ptr,wordtoreplace)))
{
while(buff_ptr < find_ptr)
fputc((int)*buff_ptr++,fp2);
fputs(replacewith,fp2);
buff_ptr += find_len;
}
fputs(buff_ptr,fp2);
}
rename(fileRepl, fileOrig);
}
fclose(fp2);
fclose(fp1);
fprintf(stderr,"Wrote conf file...\n");
return 1; // Success
}
/* Executors */
/** Executor function for Blender operations */
int exec_blender(char *input, char *output, int frame)
{
fprintf(stderr,"Preparing to execute...\n");
int ret;
char *frame_str;
char *command = "blender"; // @TODO: We currently expect this to be in PATH
char *cmd[] = { command, "-b", "-o", output, input, "-f", frame_str, (char *)NULL }; // arguments for blender
char buf[10];
struct execio *testrem;
size_t readlen;
_distren_asprintf(&frame_str, "%i", frame);
fprintf(stderr,"Executing: %s\n", frame_str);
ret = execio_open(&testrem, command, cmd);
buf[9] = '\0';
while(!execio_read(testrem, buf, 9, &readlen))
{
if(readlen > 9) {
fprintf(stderr, "Something is terribly wrong!\n");
}
buf[readlen] = '\0';
fprintf(stderr, "read \"%s\"\n", buf);
}
execio_close(testrem);
return ret;
}
void xmlinit()
{
xmlInitParser();
xmlXPathInit();
}
void xmlcleanup()
{
xmlCleanupParser();
}
/** Creates directories recursively */
int distren_mkdir_recurse(char *dirname)
{
size_t counter;
char *nextdir;
nextdir = strdup(dirname);
for(counter = 0; nextdir[counter]; counter ++)
{
/** @TODO OS-portabalize the path-separators */
if(nextdir[counter] == '/')
{
nextdir[counter] = '\0';
mkdir(nextdir, S_IRWXU | S_IRGRP | S_IROTH);
nextdir[counter] = '/';
}
}
return 0;
}
/**
@TODO: Use for constructing path to job data locally and/or on data.distren.org
*/
int job_build_path(char *filename, unsigned int jobnum)
{
return 0;
}
int downloadTar(char *url, char *destinationPath){
// Prepare to download the job tar if it isn't present
struct stat buffer;
int fstatus = stat(destinationPath, &buffer);
if(fstatus == -1)
{
// Download the Tar
if( curlget(url, destinationPath) == 0){
fprintf(stderr, "Job data retrieved successfully\n");
// free(url); @FIXME: causes doublefree! Curl must free the url?
return 0;
}
else {
fprintf(stderr, "Downloading job data from %s failed. Check your network connection.\n",url);
free(url);
return 1; // Eventually make a retry loop
}
}
else{
fprintf(stderr, "Tar already exists! Download cancelled.\n");
return 2;
}
}
int uploadOutput(char *pathtoOutput, char *urltoOutput){
if( !curlpost(pathtoOutput, urltoOutput)) // Uploads output
{
fprintf(stderr,"Upload successful, removing old output...\n");
remove(pathtoOutput); // Delete the file after its uploaded
return 0;
}
else
{
fprintf(stderr,"Upload failed. Check your network connection. Retrying upload...\n");
int tries=1;
while(tries<=10 && curlpost(pathtoOutput, urltoOutput))
{
fprintf(stderr, "Upload failed. Trying again in 10 seconds... (attempt %d of 10)\n", tries);
tries++;
sleep(10);
}
return 1; // Upload failed after multiple tries
// @FUTURE: Keep track of files that we were unable to upload, and upload them later
}
}
int unpackJob(char *pathtoTar, int jobnum){
int ret;
int buffsize = 8192;
char *buff = "";
size_t size;
struct archive_entry *ae = archive_entry_new();
struct archive *a = archive_read_new();
archive_read_support_compression_all(a);
archive_read_support_format_raw(a);
ret = archive_read_open_filename(a, pathtoTar, 16384);
if (ret != ARCHIVE_OK) {
return 1;
}
ret = archive_read_next_header(a, &ae);
if (ret != ARCHIVE_OK) {
return 1;
}
for (;;) {
size = archive_read_data(a, buff, buffsize);
if (size < 0) {
return 1;
}
if (size == 0)
break;
write(1, buff, size);
}
archive_read_finish(a);
archive_entry_free(ae);
return 0;
/*
char *tarcmd; // Command to run for tar. Migrate to libtar sometime
char *outdir; // Output Directory for tar
_distren_asprintf(&outdir, "/tmp/distren/job%d", jobnum);
mkdir("/tmp/distren", 0750); // @FIXME: Change to tmpdir once it exists
mkdir(outdir, 0750);
_distren_asprintf(&tarcmd, "tar -xvf \"%s\" -C \"%s\"", pathtoTar, outdir); // @FIXME:Use a lib here!
system(tarcmd);
free(tarcmd);
free(pathtoTar);
free(outdir);
return 0;
*/
}
void prepareJobPaths(int jobnum,char *datadir, char **urltoTar,char **pathtoTar,char **pathtoJobfile,char **urltoOutput,char **pathtoXml){
// Variable Preparation
char *jobdatapath;
_distren_asprintf(&jobdatapath, "job%d", jobnum);
_distren_asprintf(urltoTar, "http://data.distren.org/job%d/job%d.tar.gz", jobnum); // Prepares URL to download from
_distren_asprintf(pathtoTar, "%s/stor/jobdata/job%d.tar.gz", datadir, jobnum); // Prepares destination to save to
_distren_asprintf(pathtoJobfile, "%s/%s/job.blend", datadir, jobdatapath ); // Prepares the path to the jobfile
_distren_asprintf(urltoOutput, "http://data.distren.org/tmp/", jobdatapath ); // Prepares the URL where output is posted
_distren_asprintf(pathtoXml, "%s/job%d/job%d.xml", datadir, jobnum ); // Prepares the path to the job's XML file
free(jobdatapath);
}
int checkUsername(char *username){
if(username == NULL || strcmp(username, "!username") == 0 ) {
fprintf(stderr, "\nPlease ensure that your username is present in distrenslave.conf\n");
return 0;
}
else
if( username != NULL || strcmp(username, "!username") != 0 )
{
// Log in the user
if(login_user(username) == 1){
fprintf(stderr, "You should now be logged into distren.\n");
return 0;
}
else
{
fprintf(stderr, "Login failed, please check your username. If you have not registered, please do so on the DistRen website.\n");
return 1;
}
}
else
{
fprintf(stderr, "Please check your distrenslave.conf, it appears to be incorrectly formatted.\n");
return 1;
}
}
void slaveTest(){
int command;
int test = 1;
int jobnum = 0;
fprintf(stderr,"Hello!\n");
char tmpString1[100] = "";
char tmpString2[100] = "";
while(test == 1)
{
fprintf(stderr, "Welcome to DistRenSlave Alpha Interactive Test Mode\n\n");
fprintf(stderr, "\t1 \tTest posting (uploading) data\n");
fprintf(stderr, "\t2 \tTest getting frame from server\n");
fprintf(stderr, "\t3 \tTest data downloading\n");
fprintf(stderr, "\t4 \tTest archive decompression\n");
fprintf(stderr, "\t5 \tQuit\n");
scanf("%d", &command);
switch(command)
{
case 1:
fprintf(stderr,"Path to file to upload: ");
scanf("%99s", tmpString1);
fprintf(stderr,"URL to upload to: ");
scanf("%99s", tmpString2);
uploadOutput(tmpString1, tmpString2);
break;
case 2:
fprintf(stderr,"Remotio not implemented yet!\n");
break;
case 3:
fprintf(stderr,"URL to file: ");
scanf("%s", tmpString1);
fprintf(stderr,"Local save path (including filename): ");
scanf("%s", tmpString2);
if(downloadTar(tmpString1, tmpString2))
fprintf(stderr,"Error downloading!\n");
else
fprintf(stderr,"Success!\n");
break;
case 4:
fprintf(stderr,"Jobnum to decompress: ");
scanf("%d", jobnum);
fprintf(stderr,"Path to compressed data: ");
scanf("%d", tmpString2);
unpackJob(tmpString1, jobnum);
break;
case 5:
test = 0;
break;
default:
fprintf(stderr, "Invalid input, please try again.\n");
break;
}
}
}