From ddc6d175603058ee2bdf83f22fce9b119fdae82f Mon Sep 17 00:00:00 2001 From: Jan Vales Date: Mon, 30 Jan 2012 03:01:34 +0100 Subject: [PATCH 01/16] unsigned long bla = -1; ... yeah! --- prefix/prefix.c | 6 +++--- scan/scan.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/prefix/prefix.c b/prefix/prefix.c index 43e626e..57304b9 100644 --- a/prefix/prefix.c +++ b/prefix/prefix.c @@ -14,7 +14,7 @@ char* binname = "unset"; void usage () { fprintf (stderr, "\nUsage: %s [-f ] [-n ]\n", binname); fprintf (stderr, "\t -f: set the filename of the randfile. (defaults to \"numlist.bin\")\n"); - fprintf (stderr, "\t -n: set the number count to read from randfile. (defaults to -1 = read all)\n"); + fprintf (stderr, "\t -n: set the number count to read from randfile. (defaults to 0 = read all)\n"); fprintf (stderr, "\nThis application prints only TIME-INFO to stdout. Everything else goes to stderr.\n"); exit(1); } @@ -36,7 +36,7 @@ int main (int argc, char* argv[]) { double startTime, endTime; /* options */ - unsigned long n = -1; + unsigned long n = 0; char *filename = "numlist.bin"; /* store out name for usage(); */ @@ -71,7 +71,7 @@ int main (int argc, char* argv[]) { struct stat st; stat(filename, &st); size = st.st_size; - if (n == -1 || n > size){ + if (n <= 0 || n > size){ n = size; } }else{ diff --git a/scan/scan.c b/scan/scan.c index 9789cb6..04e7398 100755 --- a/scan/scan.c +++ b/scan/scan.c @@ -20,7 +20,7 @@ char* binname = "unset"; void usage () { fprintf(stderr, "\nUsage: mpirun -node 1-32 -nnp 1 %s [-f ] [-n ]\n", binname); fprintf(stderr, "\t -f: set the filename of the randfile. (defaults to \"numlist.bin\")\n"); - fprintf(stderr, "\t -n: set the number count to read from randfile. (defaults to -1 = read all)\n"); + fprintf(stderr, "\t -n: set the number count to read from randfile. (defaults to 0 = read all)\n"); #ifdef DEBUG fprintf(stderr, "***** BIG RED WARNING: COMPILED WITH XDEBUG - YOU ARE ON YOUR OWN! *****\n"); #endif -- 2.43.0 From 1171acf73c4da14db3f46d3111a27b5835915415 Mon Sep 17 00:00:00 2001 From: Jan Vales Date: Mon, 30 Jan 2012 11:30:00 +0100 Subject: [PATCH 02/16] getopt failure: now fails when reading more than $file.size from $file. --- scan/scan.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scan/scan.c b/scan/scan.c index 04e7398..aff36b0 100755 --- a/scan/scan.c +++ b/scan/scan.c @@ -29,7 +29,8 @@ void usage () { fprintf(stderr, "Compile with XDEBUG or without DEBUG to get past this limit.\n"); fprintf(stderr, "You might want to compile this without the DEBUG flag to get less noice.\n"); #endif - fprintf(stderr, "\n"); + fprintf(stderr, "\nErrorcodes:\n"); + fprintf(stderr, "\t0: Everything went OK.\n\t1: General error.\n\t2: Getopt/wrong usage error.\n\t3: Inputfile error.\n\t4: Memory error.\n\n"); } @@ -113,7 +114,8 @@ int main(int argc, char *argv[]){ file = NULL; struct stat st; stat(filename, &st); - if (size <= 0 || size > st.st_size){ + + if (size == 0){ size = st.st_size; } #ifdef DEBUG @@ -121,9 +123,17 @@ int main(int argc, char *argv[]){ if (size > 64) size = 64; #endif #endif + if (size < 0 || size > st.st_size){ + if(rank == 0){ + fprintf (stderr, "[%d/%d:%s] Cannot read %li numbers from \"%s\" - only has %li bytes.\n",rank,nodes,name,size,filename,st.st_size); + usage(); + } + MPI_Finalize(); + exit(3); + } }else{ if(rank == 0){ - fprintf (stdout, "[%d/%d:%s] File %s does not exist.\n",rank,nodes,name,filename); + fprintf (stderr, "[%d/%d:%s] File %s does not exist.\n",rank,nodes,name,filename); usage(); } MPI_Finalize(); @@ -134,18 +144,18 @@ int main(int argc, char *argv[]){ if(databuf == NULL) { fprintf(stdout, "[%d/%d:%s] malloc for databuf failed.\n",rank,nodes,name); MPI_Finalize(); - exit(2); + exit(4); } blocksize = size/nodes; if(blocksize*nodes < size){ blocksize++; if(rank == 0) - fprintf(stdout, "[%d/%d:%s] perfect split impossible: n:%d, s:%li -> bs:%li (off: %li)\n",rank,nodes,name,nodes,size,blocksize,(nodes*blocksize)-size); + fprintf(stderr, "[%d/%d:%s] perfect split impossible: n:%d, s:%li -> bs:%li (off: %li)\n",rank,nodes,name,nodes,size,blocksize,(nodes*blocksize)-size); } if(rank == 0){ - fprintf(stdout, "[%d/%d:%s] s: %li n: %d bs: %li n*b: %li n*(b+1): %li\n",rank,nodes,name,size,nodes,blocksize,nodes*blocksize, nodes*(blocksize+1)); + fprintf(stdout, "[%d/%d:%s] INFO s: %li n: %d bs: %li n*b: %li n*(b+1): %li\n",rank,nodes,name,size,nodes,blocksize,nodes*blocksize, nodes*(blocksize+1)); /* read file. */ if ((file = fopen(filename, "r"))) { @@ -154,7 +164,7 @@ int main(int argc, char *argv[]){ } fclose(file); }else{ - fprintf (stdout, "[%d/%d:%s] File %s could not be read.\n",rank,nodes,name,filename); + fprintf (stderr, "[%d/%d:%s] File %s could not be read.\n",rank,nodes,name,filename); usage(); MPI_Finalize(); exit(3); -- 2.43.0 From bb69bf6964aed63092600b94e33d818d718b9b0d Mon Sep 17 00:00:00 2001 From: Jan Vales Date: Mon, 30 Jan 2012 11:34:41 +0100 Subject: [PATCH 03/16] getopt fix: will now fail when reading more than $file.size from $file. --- prefix/prefix.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/prefix/prefix.c b/prefix/prefix.c index 57304b9..77c7f93 100644 --- a/prefix/prefix.c +++ b/prefix/prefix.c @@ -71,9 +71,13 @@ int main (int argc, char* argv[]) { struct stat st; stat(filename, &st); size = st.st_size; - if (n <= 0 || n > size){ + if (n == 0){ n = size; } + if (n < 0 || n > size){ + fprintf (stderr, "Cannot read %li numbers from \"%s\" - only has %li bytes.\n",n,filename,st.st_size); + usage(); + } }else{ fprintf (stderr, "File %s does not exist.\n", filename); usage(); -- 2.43.0 From 9abbc5022b61a2ab68821ea57ceea9ca7887b505 Mon Sep 17 00:00:00 2001 From: someone Date: Mon, 30 Jan 2012 11:59:04 +0100 Subject: [PATCH 04/16] angefangen irgendwas zu schreiben... --- _abgabe/was.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 _abgabe/was.txt diff --git a/_abgabe/was.txt b/_abgabe/was.txt new file mode 100644 index 0000000..1dac2d6 --- /dev/null +++ b/_abgabe/was.txt @@ -0,0 +1,19 @@ +was schreiben wir? + +== wir sind == + + +== openmp prefix == +einige algorithmen +graphen. + + +== mpi scan == +root -> n +berechnung +root -> n -> root + +root hat endergebnis. + +graphen. + -- 2.43.0 From cefe331ecae2fd199e3a9e32596c63ab2d6ff6b8 Mon Sep 17 00:00:00 2001 From: Jan Vales Date: Mon, 30 Jan 2012 13:17:00 +0100 Subject: [PATCH 05/16] run script for mpiscan - needs testing! --- scan/runAll.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 scan/runAll.sh diff --git a/scan/runAll.sh b/scan/runAll.sh new file mode 100755 index 0000000..7de8905 --- /dev/null +++ b/scan/runAll.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ ! -f numlist.bin ]; then + echo "generating numfile.bin" + dd if=/dev/urandom of=numlist.bin bs=1000 count=100000 +fi + +if [ ! -e build ]; then + echo "compiling stuff" + make +fi + +STARTTS="`date --iso-8601=minutes`" + +echo "STARTING: stats/raw/${STARTTS}" +mkdir -p "stats/raw/${STARTTS}" + +mpirun -node 1-32 -nnp 1 build/scan &> stats/raw/tmp && echo "done" && mv stats/raw/tmp "stats/raw/${STARTTS}/scan" + -- 2.43.0 From c03c2a1e1d09b7248189bd9d8318adc71c7563cf Mon Sep 17 00:00:00 2001 From: Jan Vales Date: Mon, 30 Jan 2012 13:19:36 +0100 Subject: [PATCH 06/16] added autocompile step. --- prefix/runAll.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/prefix/runAll.sh b/prefix/runAll.sh index 92bcf2d..79cb6ca 100755 --- a/prefix/runAll.sh +++ b/prefix/runAll.sh @@ -13,39 +13,39 @@ if [ ! -f numlist.bin ]; then dd if=/dev/urandom of=numlist.bin bs=1000 count=100000 fi +if [ ! -e build ]; then + echo "compiling" + make +fi + STARTTS="`date --iso-8601=minutes`" SCHEDULING="static dynamic guided runtime" ALGORITHMS="seq datapar recurse hillis_sum hillis_partial" -RUNLIST="$(seq 0 250000 1000000) $(seq 0 2500000 10000000)" -# $(seq 0 25000000 100000000)" +RUNLIST="$(seq 0 250000 1000000) $(seq 0 2500000 10000000) $(seq 0 25000000 100000000)" -#TODO: siehe oben echo "STARTING: stats/raw/${STARTTS}" mkdir -p "stats/raw/${STARTTS}" for sched in $SCHEDULING; do - echo "***** sched: ${sched} *****" + echo "***** sched: ${sched} *****" OMP_SCHEDULE=sched ulimit -s unlimited for algo in $ALGORITHMS; do - echo "*** algo: ${algo} ***" - for i in $RUNLIST; do + echo "*** algo: ${algo} ***" + for i in $RUNLIST; do if [ "$i" -eq "0" ] ; then echo "* ignoring 0 run *" else - build/$algo -n $i > "stats/raw/tmp" 2> /dev/null + build/$algo -n $i > "stats/raw/tmp" 2> /dev/null cat "stats/raw/tmp" mv stats/raw/tmp "stats/raw/${STARTTS}/${sched}_${algo}_n${i}" fi - done + done done done -echo "stats generation does not work - exitting" -exit 0 - cd stats ./parseDat.py -echo "done - hit ENTER to exit" -read +echo "done" + -- 2.43.0 From f7b559396d06f148809adf0064c915c02c33eae6 Mon Sep 17 00:00:00 2001 From: someone Date: Mon, 30 Jan 2012 12:11:33 +0100 Subject: [PATCH 07/16] moved merge todos to merge dir --- todo => merge/todo | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename todo => merge/todo (100%) diff --git a/todo b/merge/todo similarity index 100% rename from todo rename to merge/todo -- 2.43.0 From d01d92e72f948d29243a91d6fdc51228bb74a1ca Mon Sep 17 00:00:00 2001 From: David Kaufmann Date: Mon, 30 Jan 2012 15:10:27 +0100 Subject: [PATCH 08/16] update merge --- merge/Makefile | 2 +- merge/generate_random.sh | 5 +- merge/sort.c | 98 +++++++++++++++++++++++++++------------- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/merge/Makefile b/merge/Makefile index a442cb4..a722ecd 100644 --- a/merge/Makefile +++ b/merge/Makefile @@ -1,7 +1,7 @@ # GCC Makefile CC = gcc -CFLAGS = -pedantic -Wall -g -fopenmp -O0 --std=c99 -D_XOPEN_SOURCE +CFLAGS = -pedantic -Wall -g -fopenmp -O0 --std=c99 -D_XOPEN_SOURCE ${MYFLAGS} LDFLAGS = -fopenmp BINARIES = sort diff --git a/merge/generate_random.sh b/merge/generate_random.sh index cc88cb5..2554aa4 100755 --- a/merge/generate_random.sh +++ b/merge/generate_random.sh @@ -8,7 +8,7 @@ function pwait() { } # 10 ^ x values -X=7 +X=4 NULLEN="" PUNKTE="." for i in $(seq `expr $X - 1`); @@ -51,7 +51,8 @@ pwait rm -f unsorted1 unsorted2 rm -f numlist.h -echo "#define LISTSIZE ${VALUES}" >> numlist.h +echo "#define LISTSIZEA ${VALUES}" >> numlist.h +echo "#define LISTSIZEB ${VALUES}" >> numlist.h echo "extern int a["`expr ${VALUES} + 1`"];" >> numlist.h echo "extern int b["`expr ${VALUES} + 1`"];" >> numlist.h diff --git a/merge/sort.c b/merge/sort.c index 5e8c053..aae6c4a 100644 --- a/merge/sort.c +++ b/merge/sort.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "numlist.h" @@ -20,10 +21,10 @@ int rank(int elem, numtype * list, int len); void merge(int ti, numtype * a, int n, numtype * b, int m, numtype * c); int main ( int argc, char ** argv) { - int n = LISTSIZE, m = LISTSIZE; + int n = LISTSIZEA, m = LISTSIZEB; int p = 1; int opt, i; - int a_len, b_len, b_len_end, b_len_begin; + int block, b_len, b_len_begin, b_len_end; numtype * c; double startTime, endTime; @@ -38,45 +39,62 @@ int main ( int argc, char ** argv) { } } - //printf ("Number of processes that will be started: %i\n", p); - //printf ("----------------------------------\n"); + if (p > LISTSIZEA) { + printf ("Zu viele Aufteilungen!\n"); + exit(1); + } + if (((int)(LISTSIZEA/p))*p != LISTSIZEA) { + printf ("Liste ist nicht teilbar durch %i\n", p); + exit(1); + } + +#ifdef DEBUG + printf ("Number of processes that will be started: %i\n", p); + printf ("----------------------------------\n"); +#endif - c = (numtype *) malloc((LISTSIZE * 2 + 1) * (sizeof (numtype))); - c[LISTSIZE*2] = -1; + c = (numtype *) malloc((LISTSIZEA+LISTSIZEB) * (sizeof (numtype))); + for (i = 0; i < (LISTSIZEA+LISTSIZEB); i++) { + c[i] = 0; + } - //printlist("0 Sorted List A:", a); - //printlist("0 Sorted List B:", b); +#ifdef DEBUG + printlist("0 Sorted List A:", a, LISTSIZEA); + printlist("0 Sorted List B:", b, LISTSIZEB); +#endif startTime = omp_get_wtime(); - a_len = n/p; - #pragma omp parallel for shared(a,b,c,n,m,p,a_len) private(i,b_len_begin,b_len_end,b_len) + block = n/p; + //#pragma omp parallel for shared(a,b,c,n,m,p,block) private(i,b_len,b_len_begin,b_len_end) for (i = 0; i < p; i++) { - b_len_begin = rank(a[i*a_len], b, m); - b_len_end = rank(a[(i+1)*a_len], b, m); - if (b_len_begin < 0) { - printf ("Insert to end of list!\n"); - b_len_begin = n; - } - if (b_len_end < 0) { - //printf ("Reached end of list!\n"); - b_len_end = n; - } + b_len_begin = rank(a[i*block],b,m); + b_len_end = rank(a[(i+1)*block],b,m); b_len = b_len_end - b_len_begin; - //printf ("%i a_len: %i, b_len: %i (begin:%i ([%i]) -> end:%i [%i])\n", i+1, a_len, b_len, b_len_begin, b[b_len_begin], b_len_end, b[b_len_end]); - merge( i+1, - &a[i*a_len], - a_len, + if (i == p-1) { b_len = m-b_len_begin; } +#ifdef DEBUG + //printf ("b_len_begin: %i, b_len_end: %i, b_len: %i\n", b_len_begin, b_len_end, b_len); +#endif + + merge(i+1, + &a[i*block], + block, &b[b_len_begin], b_len, - &c[i*a_len+b_len_begin]); + &c[i*block+b_len_begin] + ); } endTime = omp_get_wtime(); printf("took %f seconds.\n", endTime-startTime); - //printlist("Sorted List:", c); - free(c); +#ifdef DEBUG + printlist("Sorted List:", c, LISTSIZEA+LISTSIZEB); + printf ("------------\n"); + merge(9, a, n, b, m, c); + printlist("Should Be: ", c, LISTSIZEA+LISTSIZEB); +#endif + //free(c); return 0; } @@ -89,14 +107,18 @@ void printlist(char * message, numtype * ptr, int len) { printf ("\n"); } +/* + * return -1 if len == 0 + * return pos if elem is to be placed before *list + * return len if elem has to be positioned after list + */ int rank(numtype elem, numtype * list, int len) { int pos; int i = len; - pos = 0; - if (elem == -1) { return -1; } + if (len == 0) { return -1; } while (i > 0) { - /*printf ("elem_list: %i %i\n", elem, *list);*/ + /*printf ("rank: %i %i\n", elem, *list);*/ if (elem <= *list) { return pos; } @@ -110,11 +132,17 @@ int rank(numtype elem, numtype * list, int len) { void merge(int ti, numtype * a, int n, numtype * b, int m, int * c) { int sum; int i; - /*printf ("sorting a:%i (%i) and b:%i (%i)\n", *a, n, *b, m);*/ +#ifdef DEBUG + //printf ("sorting a:%i (%i) and b:%i (%i)\n", *a, n, *b, m); + //printlist("sortinglista", a, n); + //printlist("sortinglistb", b, m); +#endif if (m<0) { m=0;} if (n<0) { n=0;} sum = n + m; +#ifdef DEBUG //printf ("%i modifying %i (%i+%i) (c[%i]/0x%08x -> c[%i]/0x%08x)\n", ti, sum, n, m, 0, (unsigned int) &c, sum-1, ((unsigned int) &c)+sum); +#endif for (i = 0; i < sum; i++) { // n+m == 0 if (n <= 0 && m <= 0) { @@ -122,19 +150,25 @@ void merge(int ti, numtype * a, int n, numtype * b, int m, int * c) { return; } if (n <= 0) { + //printf ("changing: c[%i] from %i to %i\n", i, c[i], *b); c[i] = *b++; m--; } else { if (m <= 0) { + //printf ("changing: c[%i] from %i to %i\n", i, c[i], *a); c[i] = *a++; n--; } else { if (*a < *b) { + //printf ("changing: c[%i] from %i to %i\n", i, c[i], *a); c[i] = *a++; n--; } else { + //printf ("changing: c[%i] from %i to %i\n", i, c[i], *b); c[i] = *b++; m--; } } } } - // printf ("merge done, n=%d, m=%d\n", n, m); +#ifdef DEBUG + //printf ("merge done, n=%d, m=%d\n", n, m); +#endif return; } -- 2.43.0 From 16075ee2679344e8e32bb47e6471d1e3c7f507ba Mon Sep 17 00:00:00 2001 From: David Kaufmann Date: Mon, 30 Jan 2012 15:17:51 +0100 Subject: [PATCH 09/16] re-add openmp --- merge/sort.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merge/sort.c b/merge/sort.c index aae6c4a..2146652 100644 --- a/merge/sort.c +++ b/merge/sort.c @@ -66,7 +66,7 @@ int main ( int argc, char ** argv) { startTime = omp_get_wtime(); block = n/p; - //#pragma omp parallel for shared(a,b,c,n,m,p,block) private(i,b_len,b_len_begin,b_len_end) + #pragma omp parallel for shared(a,b,c,n,m,p,block) private(i,b_len,b_len_begin,b_len_end) for (i = 0; i < p; i++) { b_len_begin = rank(a[i*block],b,m); b_len_end = rank(a[(i+1)*block],b,m); -- 2.43.0 From bc73ab91e781549e3600c5f33cf5974224429931 Mon Sep 17 00:00:00 2001 From: David Kaufmann Date: Mon, 30 Jan 2012 15:34:03 +0100 Subject: [PATCH 10/16] update testscript --- merge/generate_random.sh | 6 +++--- merge/test_threads.sh | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/merge/generate_random.sh b/merge/generate_random.sh index 2554aa4..6a84d25 100755 --- a/merge/generate_random.sh +++ b/merge/generate_random.sh @@ -8,7 +8,7 @@ function pwait() { } # 10 ^ x values -X=4 +X=$1 NULLEN="" PUNKTE="." for i in $(seq `expr $X - 1`); @@ -53,8 +53,8 @@ rm -f unsorted1 unsorted2 rm -f numlist.h echo "#define LISTSIZEA ${VALUES}" >> numlist.h echo "#define LISTSIZEB ${VALUES}" >> numlist.h -echo "extern int a["`expr ${VALUES} + 1`"];" >> numlist.h -echo "extern int b["`expr ${VALUES} + 1`"];" >> numlist.h +echo "extern int a[${VALUES}];" >> numlist.h +echo "extern int b[${VALUES}];" >> numlist.h rm -f numlist.c cat numlist1.c >> numlist.c diff --git a/merge/test_threads.sh b/merge/test_threads.sh index 968a7e6..9bc3012 100755 --- a/merge/test_threads.sh +++ b/merge/test_threads.sh @@ -1,13 +1,19 @@ #!/bin/bash -STATFILE=merge.dat +STATFILE=merge -rm $STATFILE -for i in 1 2 4 5 8 10 20 25 40 50 100 125 200 +rm $STATFILE.*.dat +for num in 3 4 5 6 7 do - SORTMSG=`./sort -t $i` - echo "$i:" $SORTMSG - echo $SORTMSG | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i},\1/" >> $STATFILE + ./generate_random.sh $num + make clean + make + for i in 1 2 4 5 10 20 25 50 100 + do + SORTMSG=`./sort -t $i` + echo "$i:" $SORTMSG + echo $SORTMSG | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i},\1/" >> $STATFILE.$num.dat + done done if [ $DISPLAY ]; then -- 2.43.0 From 69ad07d60d4ebbdebca8afd9e312ebb1e559c4c8 Mon Sep 17 00:00:00 2001 From: David Kaufmann Date: Mon, 30 Jan 2012 15:56:07 +0100 Subject: [PATCH 11/16] graph generator added --- merge/parseDat.py | 53 +++++++++++++++++++++++++++++++++++++++++++ merge/test_threads.sh | 16 +++++++------ 2 files changed, 62 insertions(+), 7 deletions(-) create mode 100755 merge/parseDat.py diff --git a/merge/parseDat.py b/merge/parseDat.py new file mode 100755 index 0000000..9570440 --- /dev/null +++ b/merge/parseDat.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python2 + +import os, subprocess + +def listtofile(listing, file): + fd = open(file, 'r') + for i in listing: + fd.write(i+"\n") + fd.close() + +def makeGraphs(graphs, graphtitle = "Title", xlabel = "Size of array", ylabel = "Time needed for calculation"): + gendir = "stats/generated/" + subprocess.call(["mkdir", "-p", gendir]) + p = subprocess.Popen(['gnuplot'],stdout=subprocess.PIPE,stdin=subprocess.PIPE) + # now do the plots + for graphfile in graphs: + # set output 'filename' + p.stdin.write("set terminal png font arial 8 #size 600,300\n") + p.stdin.write("set grid\n") + p.stdin.write("set datafile separator ';'\n") + p.stdin.write("set title 'Timing of each "+graphtitle+"'\n") + p.stdin.write("set xlabel '"+xlabel+"'\n") + p.stdin.write("set ylabel '"+ylabel+"'\n") + p.stdin.write("set output '"+gendir+graphfile+"'\n") + # plot 'filename' using 1:2 title 'algorithm' with lines, 'filename2' using 1:2 title 'algo2' with lines + plots = [] + for title, file, cols in graphs[graphfile]: + plots.append("'"+file+"' using "+cols+" title '"+title+"' with lines") + plotcommand = ", ".join(plots) + #print gendir+graphfile + p.stdin.write("plot "+plotcommand+"\n") + p.communicate()[0] + p.stdin.close() + +statsdir = 'stats/raw/' +runs = os.listdir(statsdir) +#print runs + +gendir = "stats/generated/" +graphs = [] +outpng = 'merge.png' +subprocess.call(["mkdir", "-p", gendir]) +for i in runs: + runfiles = os.listdir(statsdir+i) +# print runfiles + for file in runfiles: +# print "File: %s%s/%s" % (statsdir, i, file) + splitfilename = file.split('.') + dict = {} + dict['arrsize'] = splitfilename[1] + graphs.append((splitfilename[1], statsdir+i+'/'+file, '1:2')) + +makeGraphs({outpng: graphs}, "merge") diff --git a/merge/test_threads.sh b/merge/test_threads.sh index 9bc3012..85c9792 100755 --- a/merge/test_threads.sh +++ b/merge/test_threads.sh @@ -2,20 +2,22 @@ STATFILE=merge -rm $STATFILE.*.dat -for num in 3 4 5 6 7 +STARTTS="`date --iso-8601=minutes`" +mkdir -p stats/raw/${STARTTS} +EXPO="3 4 5 6 7" +PARTS="1 2 4 5 10 20 25 50 100" +for num in $EXPO do ./generate_random.sh $num make clean make - for i in 1 2 4 5 10 20 25 50 100 + for i in $PARTS do - SORTMSG=`./sort -t $i` - echo "$i:" $SORTMSG - echo $SORTMSG | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i},\1/" >> $STATFILE.$num.dat + foo=`expr 10**$num` + ./sort -t $i | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i};\1/" | tee -a stats/raw/$STARTTS/$STATFILE.$foo.dat done done if [ $DISPLAY ]; then - gnuplot merge.plt && display merge.png + ./parseDat.py fi -- 2.43.0 From e4ddb3f489a6bbdafdd963673dddacfacf7f53c9 Mon Sep 17 00:00:00 2001 From: David Kaufmann Date: Mon, 30 Jan 2012 16:04:25 +0100 Subject: [PATCH 12/16] fix filenames --- merge/test_threads.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merge/test_threads.sh b/merge/test_threads.sh index 85c9792..7848c5d 100755 --- a/merge/test_threads.sh +++ b/merge/test_threads.sh @@ -13,7 +13,7 @@ do make for i in $PARTS do - foo=`expr 10**$num` + let foo=`expr 10**$num` ./sort -t $i | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i};\1/" | tee -a stats/raw/$STARTTS/$STATFILE.$foo.dat done done -- 2.43.0 From d4da095b12473992821a06fde57b0bd2ac05ec54 Mon Sep 17 00:00:00 2001 From: David Kaufmann Date: Mon, 30 Jan 2012 16:24:32 +0100 Subject: [PATCH 13/16] optimize merge --- merge/parseDat.py | 2 +- merge/test_threads.sh | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/merge/parseDat.py b/merge/parseDat.py index 9570440..fa97ffa 100755 --- a/merge/parseDat.py +++ b/merge/parseDat.py @@ -8,7 +8,7 @@ def listtofile(listing, file): fd.write(i+"\n") fd.close() -def makeGraphs(graphs, graphtitle = "Title", xlabel = "Size of array", ylabel = "Time needed for calculation"): +def makeGraphs(graphs, graphtitle = "Title", xlabel = "Number of Parts", ylabel = "Time needed for calculation"): gendir = "stats/generated/" subprocess.call(["mkdir", "-p", gendir]) p = subprocess.Popen(['gnuplot'],stdout=subprocess.PIPE,stdin=subprocess.PIPE) diff --git a/merge/test_threads.sh b/merge/test_threads.sh index 7848c5d..fce8f99 100755 --- a/merge/test_threads.sh +++ b/merge/test_threads.sh @@ -4,17 +4,23 @@ STATFILE=merge STARTTS="`date --iso-8601=minutes`" mkdir -p stats/raw/${STARTTS} -EXPO="3 4 5 6 7" +EXPO=$(seq 1000 1000 10000) $(seq 10000 10000 100000) PARTS="1 2 4 5 10 20 25 50 100" +let maxfoo=`expr 10**8` for num in $EXPO do - ./generate_random.sh $num + echo "" + rm -f numlist.h + echo "#define LISTSIZEA ${num}" >> numlist.h + echo "#define LISTSIZEB ${num}" >> numlist.h + echo "extern int a[${maxfoo}];" >> numlist.h + echo "extern int b[${maxfoo}];" >> numlist.h + make clean make for i in $PARTS do - let foo=`expr 10**$num` - ./sort -t $i | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i};\1/" | tee -a stats/raw/$STARTTS/$STATFILE.$foo.dat + ./sort -t $i | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i};\1/" | tee -a stats/raw/$STARTTS/$STATFILE.$num.dat done done -- 2.43.0 From 07776a394abfbae9952808c574cdf5284e686182 Mon Sep 17 00:00:00 2001 From: David Kaufmann Date: Mon, 30 Jan 2012 16:46:50 +0100 Subject: [PATCH 14/16] fix merge --- merge/test_threads.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/merge/test_threads.sh b/merge/test_threads.sh index fce8f99..fd7db17 100755 --- a/merge/test_threads.sh +++ b/merge/test_threads.sh @@ -2,9 +2,13 @@ STATFILE=merge +# +# first run ./generate_random 8 +# + STARTTS="`date --iso-8601=minutes`" mkdir -p stats/raw/${STARTTS} -EXPO=$(seq 1000 1000 10000) $(seq 10000 10000 100000) +EXPO="$(seq 1000 1000 10000) $(seq 10000 10000 100000)" PARTS="1 2 4 5 10 20 25 50 100" let maxfoo=`expr 10**8` for num in $EXPO @@ -16,7 +20,6 @@ do echo "extern int a[${maxfoo}];" >> numlist.h echo "extern int b[${maxfoo}];" >> numlist.h - make clean make for i in $PARTS do -- 2.43.0 From a01422e1bdbf78f2fc5e10a0d6ced9e4824bc228 Mon Sep 17 00:00:00 2001 From: Jan Vales Date: Mon, 30 Jan 2012 18:12:48 +0100 Subject: [PATCH 15/16] i can haz graphs plz? --- scan/runAll.sh | 24 +++++++++++++++++++++++- scan/scan.c | 12 +++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/scan/runAll.sh b/scan/runAll.sh index 7de8905..c57462b 100755 --- a/scan/runAll.sh +++ b/scan/runAll.sh @@ -11,9 +11,31 @@ if [ ! -e build ]; then fi STARTTS="`date --iso-8601=minutes`" +NODELIST="$(seq 0 4 32)" +NNPLIST="$(seq 0 4 16)" +RUNLIST="$(seq 0 25000 100000) $(seq 0 250000 1000000) $(seq 0 2500000 10000000)" echo "STARTING: stats/raw/${STARTTS}" mkdir -p "stats/raw/${STARTTS}" -mpirun -node 1-32 -nnp 1 build/scan &> stats/raw/tmp && echo "done" && mv stats/raw/tmp "stats/raw/${STARTTS}/scan" +for nno in $NODELIST; do + for nnnp in $NNPLIST; do + for i in $RUNLIST; do + if [ "$nno" -eq "0" ] ; then + nno=2 + fi + if [ "$nnnp" -eq "0" ] ; then + nnnp=1 + fi + if [ "$i" -eq "0" ] ; then + echo "* ignoring 0 run *" + else + echo "running with nno:${nno} nnnp:${nnnp} n:${i}" + mpirun -node 1-${nno} -nnp ${nnnp} build/scan -n${i} &> stats/tmp && mv stats/tmp "stats/raw/${STARTTS}/no${nno}_nnp${nnnp}_n${i}" + cat "stats/raw/${STARTTS}/no${nno}_nnp${nnnp}_n${i}" | grep "timing" + fi + done + done +done + diff --git a/scan/scan.c b/scan/scan.c index aff36b0..0f5badd 100755 --- a/scan/scan.c +++ b/scan/scan.c @@ -169,23 +169,22 @@ int main(int argc, char *argv[]){ MPI_Finalize(); exit(3); } - +#ifdef DEBUG fprintf(stdout, "[%d/%d:%s] file read - distributing work.\n",rank,nodes,name); +#endif startTime = MPI_Wtime(); MPI_Isend(databuf, size-1, MPI_LONG, 0, KEY, MPI_COMM_WORLD, &request); } - /* stuff done by all nodes */ + /* receive work and propagate to next node, if not last node. */ if (startTime == 0) startTime = MPI_Wtime(); tmp = rank -1; if(tmp < 0) tmp = 0; MPI_Recv(databuf, size, MPI_LONG, tmp, KEY, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - - /* propagate data, if not last node. */ if(rank +1 < nodes) MPI_Send(databuf, size, MPI_LONG, rank+1, KEY, MPI_COMM_WORLD); prepTime = MPI_Wtime(); - /* do processing here. */ + /* do actual work here. */ for(unsigned long i = (rank*blocksize+1); i < ((rank+1)*blocksize) && i < size; i++){ databuf[i] = databuf[(i-1)] + databuf[i]; } @@ -224,14 +223,17 @@ int main(int argc, char *argv[]){ MPI_Recv(databuf, size, MPI_LONG, nodes-1, KEY, MPI_COMM_WORLD, MPI_STATUS_IGNORE); endTime = MPI_Wtime(); fprintf(stdout, "[%d/%d:%s] result: %li\n",rank,nodes,name,databuf[size-1]); + fprintf(stdout, "[%d/%d:%s] timings: prep:%f algo:%f postp:%f end:%f\n",rank,nodes,name,prepTime-startTime,algoTime-startTime,postpTime-startTime,endTime-startTime); #ifdef DEBUG fprintf(stdout, "[%d/%d:%s] res array ",rank,nodes,name); array_contents(databuf, size); #endif } +#ifdef DEBUG if(endTime == 0) endTime = MPI_Wtime(); fprintf(stdout, "[%d/%d:%s] timings: prep:%f algo:%f postp:%f end:%f\n",rank,nodes,name,prepTime-startTime,algoTime-startTime,postpTime-startTime,endTime-startTime); +#endif MPI_Finalize(); return 0; } -- 2.43.0 From 7b8a85ea46bf4709ca3522e3f87e67c784016986 Mon Sep 17 00:00:00 2001 From: Jan Vales Date: Mon, 30 Jan 2012 18:38:50 +0100 Subject: [PATCH 16/16] fix'd deadlock if there is just 1 node. --- scan/runAll.sh | 2 +- scan/scan.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scan/runAll.sh b/scan/runAll.sh index c57462b..dabdbb1 100755 --- a/scan/runAll.sh +++ b/scan/runAll.sh @@ -22,7 +22,7 @@ for nno in $NODELIST; do for nnnp in $NNPLIST; do for i in $RUNLIST; do if [ "$nno" -eq "0" ] ; then - nno=2 + nno=1 fi if [ "$nnnp" -eq "0" ] ; then nnnp=1 diff --git a/scan/scan.c b/scan/scan.c index 0f5badd..35446c1 100755 --- a/scan/scan.c +++ b/scan/scan.c @@ -48,6 +48,7 @@ int main(int argc, char *argv[]){ int tmp2; FILE *file = NULL; MPI_Request request; + MPI_Status status; unsigned long blocksize; unsigned long *databuf; @@ -216,7 +217,7 @@ int main(int argc, char *argv[]){ if(tmp >= nodes) tmp = 0; tmp2 = ((rank+1)*blocksize); if(tmp2 > size) tmp2 = size; - MPI_Send(databuf, tmp2, MPI_LONG, tmp, KEY, MPI_COMM_WORLD); + MPI_Isend(databuf, tmp2, MPI_LONG, tmp, KEY, MPI_COMM_WORLD,&request); /* receive result by root */ if(rank == 0){ @@ -234,6 +235,7 @@ int main(int argc, char *argv[]){ if(endTime == 0) endTime = MPI_Wtime(); fprintf(stdout, "[%d/%d:%s] timings: prep:%f algo:%f postp:%f end:%f\n",rank,nodes,name,prepTime-startTime,algoTime-startTime,postpTime-startTime,endTime-startTime); #endif + MPI_Wait(&request, &status); MPI_Finalize(); return 0; } -- 2.43.0