DistRen Changeset - c44707a801aa · protofusion repositories

Changeset - c44707a801aa

Parent rev.

Child rev.

[Not reviewed]

default

0 1 0

ethanzonca - 16 years ago 2009-07-06 18:02:19

Commenting, removed some lux stuff so we can focus on blender. We really need to clean up.

1 file changed with 105 insertions and 133 deletions:

src/server/distrend.c

105

133

0 comments (0 inline, 0 general)

src/server/distrend.c

➞

Show inline comments

 /*
   Copyright 2008 Nathan Phillip Brink, Ethan Zonca
   This file is a part of DistRen.
   DistRen is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.
   DistRen is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.
   You should have received a copy of the GNU Affero General Public License
   along with DistRen.  If not, see <http://www.gnu.org/licenses/>.
 */
 /* This file contains the code which both processes (renders) jobs as a slave, and the code which distributes frames to slaves after receiving them from the client portion of the codebase. */
  /* Just some notes -- Ethan Zonca
  * ++ Make data availible for other apps to parse
  * Server<==>Client Communication
  * Upload while rendering
  */
 #include <stdio.h>
 #include "execio.h"
 // needs to be implemented
 #include "options.h"
 #include <confuse.h>
 #define max 100
 // maximum number of stored jobs in memory, per job type (lux/blend). Eventually we can dump this data to disk, or atleast the remainder not in memory...
 #define max 100 // maximum number of stored jobs in memory, per job type (lux/blend). Eventually we can dump this data to disk, or atleast the remainder not in memory...
 int jobnum = 0;
-int hcfjob; //highest consecutively finished job
+int hcfjob; // Highest consecutively finished job
 int highest_jobnum; // The job number of the most recently created job, this is used when creating new jobs
 // Structures for storing job information
 // OOOOOOkay so we really need a struct for frameset rather than just an array, because we should track who each frame was sent out to. Please do this, someone! :D
 struct {
   char *name;
   char *submitter;
   char *email;
   int priority;  // 1 is lowest, 10 is highest, 0 is done
   int percent_done;
   int completed_frames; // number of completed frames
   int assigned_frames; // number of assigned frames (that are not yet completed)
   int total_frames; // how many frames are in the animation
   struct frameset **frameset;
   struct frameset **frameset; // What exactly is this now? hehe
   // Ok... what's going on with using these??? --ethanzonca
   int frame_num[];
   char slave_name[][];
   short int frame_status[];
   // For secondary upload location...
   char url;
   char login_name;
   char login_pass;
 } blendjob[max];
 /*
 // -- Matthew's handywork --
 // struct for storing information on each frame for a particular blender job
 struct {
 	int frame_num; // frame number to render
 	char slave_name; // user that frame is assigned to
 	int frame_status; // status of frame, 0= unassigned, 1= taken, 2= done
 } frameset[]; // Frameset array is generated by status_report_generator function
                          // Using this method to save memory, because if animation starts on a high frame number, it would waste a lot of RAM on empty structures
 // -- end of Matthew's handywork --
 */
 struct {
   char *name;
   char *submitter;
   char *email;
   int priority;
   int mode; // 0 = Static Render, stop at Spp, or infinity if spp is null. Framerange is ignored || 1 = animation, stop at Spp and stay in framerange.
   int spp;
   struct frameset ** frameset;
 } luxjob[max];
 int main(int argc, char *argv[])
+{
 // I'm assuming this is a non-working framework :)
 	int distrend_do_config(int argc, char *argv[], struct distrend_config *config)
+	{
 	  cfg_opt_t myopts =
+	    {
 	      CFG_SEC("listen",  /* this must be imported into struct listens (which must still be declared) */
+	    {
 	      CFG_SIMPLE_STR("type", NULL),
 		CFG_SIMPLE_STR("path", NULL),
 		CFG_SIMPLE_INT("port", NULL)
 		},
 		      CFGF_MULTI),
 	      CFG_END
 	    };
 	  config = malloc(sizeof(struct distrend_config));
 	  options_init(argc, argv, &config->mycfg, &myopts, "server", &config->options);
 	  return 0;
+	}
 	int distrend_config_free(struct distrend_config *config)
+	{
 	  options_free(config->options);
 	  free(config);
 	  return 0;
+	}
   int cont = 1;
   struct distrend_listenset *listenset;
   struct distrend_config *config;
   distrend_do_config(argc, argv, &config);
   distrend_listen(&listenset, config);
   /* This is called the ``main loop'' */
   while(cont)
+    {
       struct distren_action *action;
       distrend_accept(&action);
       cont = distrend_do(action);
       distrend_action_free(action);
+    }
   distrend_unlisten(listenset);
   distrend_config_free(config);
   return 0;
+}
 struct distrend_config
+{
   cfg_t *mycfg;
   struct options_common *options;
   struct distrend_listen **listens; /*< Null terminated array of structs */
+}
 /* Notes on Registration:
 We need to generate a key in registeruser() that is returned to the client.
 How will we transfer the key?
 */
 // Registration on server. Needs attention. Prevent account spamming.
 // Set up something like: distrend -c username email@example.com
 void registeruser(char *username, char *email){
   // Logs into sandboxed user on zserver2 and registers a user. Should eventually generate a key on the server and return it to the user.
   // All created user accounts should be sandboxed accordingly, requiring a different skel, and the default shell to be rbash. Also,
   // a custom path defined in the .bash_profile of the skel is needed.
   char buf[10];
   struct execio *testrem;
   char *execargv[] =
+    {
   "ssh",
   "distren_setup@protofusion.org",
   "-i",
   "setup.rsa", // default distributed key, account can only create users.
   "-p",
   "23",
   "sudo /usr/sbin/useradd",
   "-M",
   "-c",
   email,
   "-d",
   "/home/distren",
   "--gid",
   "541", // Add in shellscript to generate ssh key and return it to the user somehow
   username,
   (char *)NULL
   };
   size_t readlen;
   fprintf(stderr, "Opening stream:\n", execio_open(&testrem, "ssh", execargv));
   buf[9] = '\0'; // null-terminating the array...
   while(!execio_read(testrem, buf, 9, &readlen)) // What's with the readlen stuff?
+    {
       if(readlen > 9) {
 	fprintf(stderr, "!!!! Something is terribly wrong!\n");
+      }
       if(buf == 0) {
 	fprintf(stderr, "Make this throw a successfull message if no output is returned... maybe");
+      }
     buf[readlen] = '\0'; // Null-terminating the end of it again based on how large the data is?
     fprintf(stderr, "read \"%s\"\n", buf);
+    }
   execio_close(testrem);
+}
 void loginuser(char *username, int secret){
   // Logs into sandboxed user on zserver2 as a client, currently does nothing
   char buf[10];
   struct execio *testrem;
   char *execargv[] =
+    {
   "ssh",
   "username@protofusion.org", // username must be read from the conf
   "-i",
   "username.rsa", // Key created from registeruser()
   "-p",
   "23",
   "echo",
   "hello", // This should eventually open a non-terminating connection to the server for communication
   (char *)NULL
   };
   size_t readlen;
   fprintf(stderr, "Opening stream:\n", execio_open(&testrem, "ssh", execargv));
   buf[9] = '\0'; // null-terminating the array...
   while(!execio_read(testrem, buf, 9, &readlen)) // What's with the readlen stuff?
+    {
       if(readlen > 9) {
 	fprintf(stderr, "!!!! Something is terribly wrong!\n");
+      }
       if(buf == 0) {
 	fprintf(stderr, "Put stuff here-ish to check whether the operation(s) were successful.");
+      }
     buf[readlen] = '\0'; // Null-terminating the end of it again based on how large the data is?
     fprintf(stderr, "read \"%s\"\n", buf);
+    }
   execio_close(testrem);
+}
 /*
  frame[frame] Assignments:
   "NULL" - don't render me
   "0" - cancelled
   "1" - unassigned
   "2" - assigned to slave
   "3" - completed by slave and uploaded
  Have a script crawl through each job in the arrays, ordered by priority?, and assign a certain number of frames to each slave.
  Then we will need some sort of watchdog to monitor slaves on the main server to check for stales.
  Have a script crawl through each job in the arrays, priority-biased, and assign a frame to each slave.
  Then we will need some sort of watchdog to monitor slaves on the main server to check for stales. Maybe not worry about this for now.
 */
 /* -- Coding using OLD FRAMESET STRUCTURE... temporarily saving it!! --
+/* -- Coding using OLD FRAMESET STRUCTURE... temporarily saving it!! -- :: Normaldotcom glances at lordofwar with a concerned look...
 // This function creates frame array based on the total number of frames to be rendered, which will then be parsed by function frame_farmer.
 void frame_num_struct_builder(int sframe, int eframe) {
 	int jobnum_new = highest_jobnum + 1;
 	int total = (sframe - eframe) +1;  // total number of frames
 	int fcount = sframe; // Used to create all the frames in the structure from sframe to eframe
 	int x = 0;
 	blendjob[jobnum_new].total_frames = total; // sets the total number of frames in animation for status purposes
 	while(x < total){
 		blendjob[jobnum_new].frameset[x].frame_num = fcount;
 		x++;
 		fcount++;
+	}
 	highest_jobnum++;
+}
 // matches your computer up with a lovely frame to render
 int frame_finder(){
 	int your_frame = NULL;  // your_frame is an interger value that will be given to the client as the frame number to render
 	int finder_jobnum = 0;
 	int frameset_count = 0; // the frameset number, note* frames in an animation don't start at zero
 	short int done = 0;
 	int priority = 10;
 	while(priority >= 1){ // start the scan for the next job with the highest priority
 		finder_jobnum = hcfjob + 1; // reset it to start scanning at first uncompleted job for the pass at each priority level
 		while(finder_jobnum <= highest_jobnum){
 			if(blendjob[finder_jobnum].priority = priority){  // looks for a job with the current priority value
 				done = 1;									  // notice it starts by looking at the oldest job first
 				break;
+			}
 			if((done) = 1)  // If it has found a job with the current priority value, it will break out of the loop
 				break;    // If none is found it goes to the next job to see if it is of the current priority value
 			else
 				finder_jobnum++;
+		}
 		if(done = 1) // if job has been found, it lets it out of the priority changer loop
 			break;
 		priority--;
+	}
 	while(your_frame < blendjob[finder_jobnum].total_frames){ // Finds the frameset number with a frame that needs to be rendered
 		if (blendjob[finder_jobnum].frameset[frameset_count].frame_status = 0)  // If frame that is not assigned has been found, frameset_count is not changed
 			break;																// and frameset_count is used to give the frame number later in this funciton
 		x++;  // If frame is assigned or done, it goes to next frame
+	}
 blendjob[jobnum].frameset[x].frame_status++; // sets the value of the frame to 2, which means its taken
 your_frame = blendjob[jobnum].frameset[x].frame_num; //  Takes the frameset found in the while statement above, and extracts the frame number from it and assigns it to the int your_frame
 if(your_frame = null)  // If that job had no open frames for some reason, run the status report generator so that
 	status_report_generator();  //the job priority can be changed to 0
 return your_frame; // your_frame is returned as the frame to be rendered
+}
 // This figures out how much of the job is done, where jobnum corresponds to the job number
 // This uses pointers, so when it is run it updates certain values in memory
 void status_report_generator(){
 	while(blendjob[(hcfjob+1)].priority = 0)  //If the job after the highest consecutively finished job is finished
 		hcfjob+1;  // adds 1 to the highest consecutively finished job and checks the next one, till the job after the hcfjob is not done
 	int num1 = hcfjob+1; // to scan through jobs
 	int num2 = 0;		 // to scan through frames
 	while(num1 <= highest_jobnum){
 			if(blendjob[num1].frameset[num2].priority != 0){ // If the job is not done, scan it
 				float finished_frames = 0; // variable that counts the completed frames
 				int pending_frames = 0; // variable that counts the assigned frames
 				float percent = 0;  // variable that stores the percent done of the blendjob
 				while(num2 <= blendjob[jobnum].total_frames){ // If
 					if(blendjob[jobnum].frameset[num2].frame_status = 2)
 						finished_frames++;
 					if(blendjob[jobnum].frameset[num2].frame_status = 1)
 						pending_frames++;
 					num2++;
+				}
 				percent = (finished_frames / blendjob[num1].total_frames) * 100;
 				blendjob[num1].completed_frames = finished_frames;
 				blendjob[num1].assigned_frames = pending_frames;
 				blendjob[num1].percent_done = percent;
+			}
 			num1++;
+		}
+	}
 // This function makes the value of the frame 2, which means its completed.
 void the_finisher(int frame){
 	blendjob[jobnum].frameset[frame].frame_status++;
+}
 */ // --End of saved frameset structure using code --
 // -- NEW CODE TO USE ARRAYS INSTEAD OF THE FRAMESET STRUCTURE --
 // This function creates frame array based on the total number of frames to be rendered, which will then be parsed by function frame_farmer.
 void frame_num_struct_builder(int sframe, int eframe) {
 	int jobnum_new = highest_jobnum + 1;
 	int total = (sframe - eframe) +1;  // total number of frames
 	int fcount = sframe; // Used to create all the frames in the structure from sframe to eframe
 	int x = 0;
 	blendjob[jobnum_new].total_frames = total; // sets the total number of frames in animation for status purposes
 	while(x < total){
 		blendjob[jobnum_new].frame_num[x] = fcount;
 		x++;
 		fcount++;
+	}
 	highest_jobnum++;
+}
 // matches your computer up with a lovely frame to render
 int frame_finder(){
 	int your_frame = NULL;  // your_frame is an interger value that will be given to the client as the frame number to render
 	int finder_jobnum = 0;
 	int frameset_count = 0; // the frameset number, note* frames in an animation don't start at zero
 	short int done = 0;
 	int priority = 10;
 	while(priority >= 1){ // start the scan for the next job with the highest priority
 		finder_jobnum = hcfjob + 1; // reset it to start scanning at first uncompleted job for the pass at each priority level
 		while(finder_jobnum <= highest_jobnum){
 			if(blendjob[finder_jobnum].priority = priority){  // looks for a job with the current priority value
 				done = 1;									  // notice it starts by looking at the oldest job first
 				break;
+			}
 			if((done) = 1)  // If it has found a job with the current priority value, it will break out of the loop
 				break;    // If none is found it goes to the next job to see if it is of the current priority value
 			else
 				finder_jobnum++;
+		}
 		if(done = 1) // if job has been found, it lets it out of the priority changer loop
 			break;
 		priority--;
+	}
 	while(your_frame < blendjob[finder_jobnum].total_frames){ // Finds the frameset number with a frame that needs to be rendered
 		if (blendjob[finder_jobnum].frame_status[frameset_count] = 0)  // If frame that is not assigned has been found, frameset_count is not changed
 			break;																// and frameset_count is used to give the frame number later in this funciton
 		x++;  // If frame is assigned or done, it goes to next frame
+	}
 blendjob[jobnum].frame_status[x]++; // sets the value of the frame to 2, which means its taken
 your_frame = blendjob[jobnum].frame_num[x]; //  Takes the frameset found in the while statement above, and extracts the frame number from it and assigns it to the int your_frame
 if(your_frame = null)  // If that job had no open frames for some reason, run the status report generator so that
 	status_report_generator();  //the job priority can be changed to 0
 return your_frame; // your_frame is returned as the frame to be rendered
+}
 // This figures out how much of the job is done, where jobnum corresponds to the job number
 // This uses pointers, so when it is run it updates certain values in memory
 void status_report_generator(){
 	while(blendjob[(hcfjob+1)].priority = 0)  //If the job after the highest consecutively finished job is finished
 		hcfjob+1;  // adds 1 to the highest consecutively finished job and checks the next one, till the job after the hcfjob is not done
 	int num1 = hcfjob+1; // to scan through jobs
 	int num2 = 0;		 // to scan through frames
 	while(num1 <= highest_jobnum){
 			if(blendjob[num1].priority != 0){ // If the job is not done, scan it
 				float finished_frames = 0; // variable that counts the completed frames
 				int pending_frames = 0; // variable that counts the assigned frames
 				float percent = 0;  // variable that stores the percent done of the blendjob
 				while(num2 <= blendjob[jobnum].total_frames){ // If
 					if(blendjob[jobnum].frame_status[num2] = 2)
 						finished_frames++;
 					if(blendjob[jobnum].frame_status[num2] = 1)
 						pending_frames++;
 					num2++;
+				}
 				percent = (finished_frames / blendjob[num1].total_frames) * 100;
 				blendjob[num1].completed_frames = finished_frames;
 				blendjob[num1].assigned_frames = pending_frames;
 				blendjob[num1].percent_done = percent;
+			}
 			num1++;
+		}
+	}
 // This function makes the value of the frame 2, which means its completed.
 void the_finisher(int frame){
 	blendjob[jobnum].frame_status[frame]++;
+}
 // Queuer: this function should add files to the queue
 // Type: 0 = blender, 1 = luxrender
 // jobnum is the current job number
 void queue(int type, char *name, char *submitter, char *email, int priority, int mode, int spp, *frameset) {
   if(type == 1){
     blendjob[jobnum].name = name;
     blendjob[jobnum].submitter = submitter;
     blendjob[jobnum].email = email;
     blendjob[jobnum].priority = priority;
     blendjob[jobnum].frameset =  frameset;
+  }
   if(type == 2){
     luxjob[jobnum].name = name;
     luxjob[jobnum].submitter = submitter;
     luxjob[jobnum].email = email;
     luxjob[jobnum].priority = priority;
     luxjob[jobnum].mode = mode;
     luxjob[jobnum].spp = spp;
     luxjob[jobnum].frameset = frameset;
     // handle lux modes somehow.
+}
   else{
     // Throw error.
+  }
 jobnum++;
+}
 /*
 Slave execution code lies below
  * --------------------------------------------------------------------
  * Slave code resides below.
+ *
  * Slave listens on server for a command in the format of each function...
  * We need if's for returns... ==> watchdog
 */
 Slave listens on server for a command in the format of each function...
 We need if's for returns... ==> watchdog
 */
 // Registration on server. Needs attention. Prevent account spamming.
 // Key transfer?
 // Set up something like: distrend -c username email@example.com
 void registeruser(char *username, char *email){
   // Logs into sandboxed user on zserver2 and registers a user. Should eventually generate a key on the server and return it to the user.
   // All created user accounts should be sandboxed accordingly, requiring a different skel, and the default shell to be rbash. Also,
   // a custom path defined in the .bash_profile of the skel is needed.
   char buf[10];
   struct execio *testrem;
   char *execargv[] =
+    {
   "ssh",
   "distren_setup@protofusion.org",
   "-i",
   "setup.rsa", // default distributed key, account can only create users.
   "-p",
   "23",
   "sudo /usr/sbin/useradd",
   "-M",
   "-c",
   email,
   "-d",
   "/home/distren",
   "--gid",
   "541", // Add in shellscript to generate ssh key and return it to the user somehow
   username,
   (char *)NULL
   };
   size_t readlen;
   fprintf(stderr, "Opening stream:\n", execio_open(&testrem, "ssh", execargv));
   buf[9] = '\0'; // null-terminating the array...
   while(!execio_read(testrem, buf, 9, &readlen)) // What's with the readlen stuff?
+    {
       if(readlen > 9) {
 	fprintf(stderr, "!!!! Something is terribly wrong!\n");
+      }
       if(buf == 0) {
 	fprintf(stderr, "Make this throw a successfull message if no output is returned... maybe");
+      }
     buf[readlen] = '\0'; // Null-terminating the end of it again based on how large the data is?
     fprintf(stderr, "read \"%s\"\n", buf);
+    }
   execio_close(testrem);
+}
 void loginuser(char *username, int secret){
   // Logs into sandboxed user on zserver2 as a client, currently does nothing
   char buf[10];
   struct execio *testrem;
   char *execargv[] =
+    {
   "ssh",
   "username@protofusion.org", // username must be read from the conf
   "-i",
   "username.rsa", // Key created from registeruser()
   "-p",
   "23",
   "echo",
   "hello", // This should eventually open a non-terminating connection to the server for communication
   (char *)NULL
   };
   size_t readlen;
   fprintf(stderr, "Opening stream:\n", execio_open(&testrem, "ssh", execargv));
   buf[9] = '\0'; // null-terminating the array...
   while(!execio_read(testrem, buf, 9, &readlen)) // What's with the readlen stuff?
+    {
       if(readlen > 9) {
 	fprintf(stderr, "!!!! Something is terribly wrong!\n");
+      }
       if(buf == 0) {
 	fprintf(stderr, "Put stuff here-ish to check whether the operation(s) were successful.");
+      }
     buf[readlen] = '\0'; // Null-terminating the end of it again based on how large the data is?
     fprintf(stderr, "read \"%s\"\n", buf);
+    }
   execio_close(testrem);
+}
 // Executors
 void exec_blender(char *input, char *output, int frame) {
   int ret;
   /* SEGFAULTAGE :-D */
   char *cmd[] = { "blender", "-b", "-o", output, input, "-f", frame, (char *)0 };
   ret = execv("/usr/bin/blender", cmd);
   // OHNOBINKI! ... check this... Its supposed to send a command back to the server and run the_finisher(); function which sets the frame status to complete.
   fprintf(stdin, the_finisher(frame));
 void exec_luxrender_single(char *input, char *output) {
   int ret;
   char *cmd[] = { "luxrender", "-something", "something", "something", (char *)0 };
   ret = execv("/usr/bin/luxrender", cmd);
+}
 void exec_luxrender_anim(char *input, char *output, int sframe, int eframe, int spp) {
 // spp is samples per pixel limitation on each frame
   int ret;
   char *cmd[] = { "luxrender", "-something", "something", "something", (char *)0 };
   ret = execv ("/usr/bin/luxrender", cmd);
+}
   return 0;
+}

0 comments (0 inline, 0 general)