From ddc6d175603058ee2bdf83f22fce9b119fdae82f Mon Sep 17 00:00:00 2001
From: Jan Vales <e0726236@jupiter.par.tuwien.ac.at>
Date: Mon, 30 Jan 2012 03:01:34 +0100
Subject: [PATCH 01/16] unsigned long bla = -1; ... yeah!
---
prefix/prefix.c | 6 +++---
scan/scan.c | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/prefix/prefix.c b/prefix/prefix.c
index 43e626e..57304b9 100644
--- a/prefix/prefix.c
+++ b/prefix/prefix.c
@@ -14,7 +14,7 @@ char* binname = "unset";
void usage () {
fprintf (stderr, "\nUsage: %s [-f <string>] [-n <number>]\n", binname);
fprintf (stderr, "\t -f: set the filename of the randfile. (defaults to \"numlist.bin\")\n");
- fprintf (stderr, "\t -n: set the number count to read from randfile. (defaults to -1 = read all)\n");
+ fprintf (stderr, "\t -n: set the number count to read from randfile. (defaults to 0 = read all)\n");
fprintf (stderr, "\nThis application prints only TIME-INFO to stdout. Everything else goes to stderr.\n");
exit(1);
}
@@ -36,7 +36,7 @@ int main (int argc, char* argv[]) {
double startTime, endTime;
/* options */
- unsigned long n = -1;
+ unsigned long n = 0;
char *filename = "numlist.bin";
/* store out name for usage(); */
@@ -71,7 +71,7 @@ int main (int argc, char* argv[]) {
struct stat st;
stat(filename, &st);
size = st.st_size;
- if (n == -1 || n > size){
+ if (n <= 0 || n > size){
n = size;
}
}else{
diff --git a/scan/scan.c b/scan/scan.c
index 9789cb6..04e7398 100755
--- a/scan/scan.c
+++ b/scan/scan.c
@@ -20,7 +20,7 @@ char* binname = "unset";
void usage () {
fprintf(stderr, "\nUsage: mpirun -node 1-32 -nnp 1 %s [-f <string>] [-n <number>]\n", binname);
fprintf(stderr, "\t -f: set the filename of the randfile. (defaults to \"numlist.bin\")\n");
- fprintf(stderr, "\t -n: set the number count to read from randfile. (defaults to -1 = read all)\n");
+ fprintf(stderr, "\t -n: set the number count to read from randfile. (defaults to 0 = read all)\n");
#ifdef DEBUG
fprintf(stderr, "***** BIG RED WARNING: COMPILED WITH XDEBUG - YOU ARE ON YOUR OWN! *****\n");
#endif
--
2.43.0
From 1171acf73c4da14db3f46d3111a27b5835915415 Mon Sep 17 00:00:00 2001
From: Jan Vales <e0726236@jupiter.par.tuwien.ac.at>
Date: Mon, 30 Jan 2012 11:30:00 +0100
Subject: [PATCH 02/16] getopt failure: now fails when reading more than
$file.size from $file.
---
scan/scan.c | 24 +++++++++++++++++-------
1 file changed, 17 insertions(+), 7 deletions(-)
diff --git a/scan/scan.c b/scan/scan.c
index 04e7398..aff36b0 100755
--- a/scan/scan.c
+++ b/scan/scan.c
@@ -29,7 +29,8 @@ void usage () {
fprintf(stderr, "Compile with XDEBUG or without DEBUG to get past this limit.\n");
fprintf(stderr, "You might want to compile this without the DEBUG flag to get less noice.\n");
#endif
- fprintf(stderr, "\n");
+ fprintf(stderr, "\nErrorcodes:\n");
+ fprintf(stderr, "\t0: Everything went OK.\n\t1: General error.\n\t2: Getopt/wrong usage error.\n\t3: Inputfile error.\n\t4: Memory error.\n\n");
}
@@ -113,7 +114,8 @@ int main(int argc, char *argv[]){
file = NULL;
struct stat st;
stat(filename, &st);
- if (size <= 0 || size > st.st_size){
+
+ if (size == 0){
size = st.st_size;
}
#ifdef DEBUG
@@ -121,9 +123,17 @@ int main(int argc, char *argv[]){
if (size > 64) size = 64;
#endif
#endif
+ if (size < 0 || size > st.st_size){
+ if(rank == 0){
+ fprintf (stderr, "[%d/%d:%s] Cannot read %li numbers from \"%s\" - only has %li bytes.\n",rank,nodes,name,size,filename,st.st_size);
+ usage();
+ }
+ MPI_Finalize();
+ exit(3);
+ }
}else{
if(rank == 0){
- fprintf (stdout, "[%d/%d:%s] File %s does not exist.\n",rank,nodes,name,filename);
+ fprintf (stderr, "[%d/%d:%s] File %s does not exist.\n",rank,nodes,name,filename);
usage();
}
MPI_Finalize();
@@ -134,18 +144,18 @@ int main(int argc, char *argv[]){
if(databuf == NULL) {
fprintf(stdout, "[%d/%d:%s] malloc for databuf failed.\n",rank,nodes,name);
MPI_Finalize();
- exit(2);
+ exit(4);
}
blocksize = size/nodes;
if(blocksize*nodes < size){
blocksize++;
if(rank == 0)
- fprintf(stdout, "[%d/%d:%s] perfect split impossible: n:%d, s:%li -> bs:%li (off: %li)\n",rank,nodes,name,nodes,size,blocksize,(nodes*blocksize)-size);
+ fprintf(stderr, "[%d/%d:%s] perfect split impossible: n:%d, s:%li -> bs:%li (off: %li)\n",rank,nodes,name,nodes,size,blocksize,(nodes*blocksize)-size);
}
if(rank == 0){
- fprintf(stdout, "[%d/%d:%s] s: %li n: %d bs: %li n*b: %li n*(b+1): %li\n",rank,nodes,name,size,nodes,blocksize,nodes*blocksize, nodes*(blocksize+1));
+ fprintf(stdout, "[%d/%d:%s] INFO s: %li n: %d bs: %li n*b: %li n*(b+1): %li\n",rank,nodes,name,size,nodes,blocksize,nodes*blocksize, nodes*(blocksize+1));
/* read file. */
if ((file = fopen(filename, "r"))) {
@@ -154,7 +164,7 @@ int main(int argc, char *argv[]){
}
fclose(file);
}else{
- fprintf (stdout, "[%d/%d:%s] File %s could not be read.\n",rank,nodes,name,filename);
+ fprintf (stderr, "[%d/%d:%s] File %s could not be read.\n",rank,nodes,name,filename);
usage();
MPI_Finalize();
exit(3);
--
2.43.0
From bb69bf6964aed63092600b94e33d818d718b9b0d Mon Sep 17 00:00:00 2001
From: Jan Vales <e0726236@jupiter.par.tuwien.ac.at>
Date: Mon, 30 Jan 2012 11:34:41 +0100
Subject: [PATCH 03/16] getopt fix: will now fail when reading more than
$file.size from $file.
---
prefix/prefix.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/prefix/prefix.c b/prefix/prefix.c
index 57304b9..77c7f93 100644
--- a/prefix/prefix.c
+++ b/prefix/prefix.c
@@ -71,9 +71,13 @@ int main (int argc, char* argv[]) {
struct stat st;
stat(filename, &st);
size = st.st_size;
- if (n <= 0 || n > size){
+ if (n == 0){
n = size;
}
+ if (n < 0 || n > size){
+ fprintf (stderr, "Cannot read %li numbers from \"%s\" - only has %li bytes.\n",n,filename,st.st_size);
+ usage();
+ }
}else{
fprintf (stderr, "File %s does not exist.\n", filename);
usage();
--
2.43.0
From 9abbc5022b61a2ab68821ea57ceea9ca7887b505 Mon Sep 17 00:00:00 2001
From: someone <someone@somenet.org>
Date: Mon, 30 Jan 2012 11:59:04 +0100
Subject: [PATCH 04/16] angefangen irgendwas zu schreiben...
---
_abgabe/was.txt | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
create mode 100644 _abgabe/was.txt
diff --git a/_abgabe/was.txt b/_abgabe/was.txt
new file mode 100644
index 0000000..1dac2d6
--- /dev/null
+++ b/_abgabe/was.txt
@@ -0,0 +1,19 @@
+was schreiben wir?
+
+== wir sind ==
+
+
+== openmp prefix ==
+einige algorithmen
+graphen.
+
+
+== mpi scan ==
+root -> n
+berechnung
+root -> n -> root
+
+root hat endergebnis.
+
+graphen.
+
--
2.43.0
From cefe331ecae2fd199e3a9e32596c63ab2d6ff6b8 Mon Sep 17 00:00:00 2001
From: Jan Vales <e0726236@jupiter.par.tuwien.ac.at>
Date: Mon, 30 Jan 2012 13:17:00 +0100
Subject: [PATCH 05/16] run script for mpiscan - needs testing!
---
scan/runAll.sh | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
create mode 100755 scan/runAll.sh
diff --git a/scan/runAll.sh b/scan/runAll.sh
new file mode 100755
index 0000000..7de8905
--- /dev/null
+++ b/scan/runAll.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ ! -f numlist.bin ]; then
+ echo "generating numfile.bin"
+ dd if=/dev/urandom of=numlist.bin bs=1000 count=100000
+fi
+
+if [ ! -e build ]; then
+ echo "compiling stuff"
+ make
+fi
+
+STARTTS="`date --iso-8601=minutes`"
+
+echo "STARTING: stats/raw/${STARTTS}"
+mkdir -p "stats/raw/${STARTTS}"
+
+mpirun -node 1-32 -nnp 1 build/scan &> stats/raw/tmp && echo "done" && mv stats/raw/tmp "stats/raw/${STARTTS}/scan"
+
--
2.43.0
From c03c2a1e1d09b7248189bd9d8318adc71c7563cf Mon Sep 17 00:00:00 2001
From: Jan Vales <e0726236@jupiter.par.tuwien.ac.at>
Date: Mon, 30 Jan 2012 13:19:36 +0100
Subject: [PATCH 06/16] added autocompile step.
---
prefix/runAll.sh | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/prefix/runAll.sh b/prefix/runAll.sh
index 92bcf2d..79cb6ca 100755
--- a/prefix/runAll.sh
+++ b/prefix/runAll.sh
@@ -13,39 +13,39 @@ if [ ! -f numlist.bin ]; then
dd if=/dev/urandom of=numlist.bin bs=1000 count=100000
fi
+if [ ! -e build ]; then
+ echo "compiling"
+ make
+fi
+
STARTTS="`date --iso-8601=minutes`"
SCHEDULING="static dynamic guided runtime"
ALGORITHMS="seq datapar recurse hillis_sum hillis_partial"
-RUNLIST="$(seq 0 250000 1000000) $(seq 0 2500000 10000000)"
-# $(seq 0 25000000 100000000)"
+RUNLIST="$(seq 0 250000 1000000) $(seq 0 2500000 10000000) $(seq 0 25000000 100000000)"
-#TODO: siehe oben
echo "STARTING: stats/raw/${STARTTS}"
mkdir -p "stats/raw/${STARTTS}"
for sched in $SCHEDULING; do
- echo "***** sched: ${sched} *****"
+ echo "***** sched: ${sched} *****"
OMP_SCHEDULE=sched
ulimit -s unlimited
for algo in $ALGORITHMS; do
- echo "*** algo: ${algo} ***"
- for i in $RUNLIST; do
+ echo "*** algo: ${algo} ***"
+ for i in $RUNLIST; do
if [ "$i" -eq "0" ] ; then
echo "* ignoring 0 run *"
else
- build/$algo -n $i > "stats/raw/tmp" 2> /dev/null
+ build/$algo -n $i > "stats/raw/tmp" 2> /dev/null
cat "stats/raw/tmp"
mv stats/raw/tmp "stats/raw/${STARTTS}/${sched}_${algo}_n${i}"
fi
- done
+ done
done
done
-echo "stats generation does not work - exitting"
-exit 0
-
cd stats
./parseDat.py
-echo "done - hit ENTER to exit"
-read
+echo "done"
+
--
2.43.0
From f7b559396d06f148809adf0064c915c02c33eae6 Mon Sep 17 00:00:00 2001
From: someone <someone@somenet.org>
Date: Mon, 30 Jan 2012 12:11:33 +0100
Subject: [PATCH 07/16] moved merge todos to merge dir
---
todo => merge/todo | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename todo => merge/todo (100%)
diff --git a/todo b/merge/todo
similarity index 100%
rename from todo
rename to merge/todo
--
2.43.0
From d01d92e72f948d29243a91d6fdc51228bb74a1ca Mon Sep 17 00:00:00 2001
From: David Kaufmann <astra@ionic.at>
Date: Mon, 30 Jan 2012 15:10:27 +0100
Subject: [PATCH 08/16] update merge
---
merge/Makefile | 2 +-
merge/generate_random.sh | 5 +-
merge/sort.c | 98 +++++++++++++++++++++++++++-------------
3 files changed, 70 insertions(+), 35 deletions(-)
diff --git a/merge/Makefile b/merge/Makefile
index a442cb4..a722ecd 100644
--- a/merge/Makefile
+++ b/merge/Makefile
@@ -1,7 +1,7 @@
# GCC Makefile
CC = gcc
-CFLAGS = -pedantic -Wall -g -fopenmp -O0 --std=c99 -D_XOPEN_SOURCE
+CFLAGS = -pedantic -Wall -g -fopenmp -O0 --std=c99 -D_XOPEN_SOURCE ${MYFLAGS}
LDFLAGS = -fopenmp
BINARIES = sort
diff --git a/merge/generate_random.sh b/merge/generate_random.sh
index cc88cb5..2554aa4 100755
--- a/merge/generate_random.sh
+++ b/merge/generate_random.sh
@@ -8,7 +8,7 @@ function pwait() {
}
# 10 ^ x values
-X=7
+X=4
NULLEN=""
PUNKTE="."
for i in $(seq `expr $X - 1`);
@@ -51,7 +51,8 @@ pwait
rm -f unsorted1 unsorted2
rm -f numlist.h
-echo "#define LISTSIZE ${VALUES}" >> numlist.h
+echo "#define LISTSIZEA ${VALUES}" >> numlist.h
+echo "#define LISTSIZEB ${VALUES}" >> numlist.h
echo "extern int a["`expr ${VALUES} + 1`"];" >> numlist.h
echo "extern int b["`expr ${VALUES} + 1`"];" >> numlist.h
diff --git a/merge/sort.c b/merge/sort.c
index 5e8c053..aae6c4a 100644
--- a/merge/sort.c
+++ b/merge/sort.c
@@ -2,6 +2,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
+#include <limits.h>
#include <omp.h>
#include "numlist.h"
@@ -20,10 +21,10 @@ int rank(int elem, numtype * list, int len);
void merge(int ti, numtype * a, int n, numtype * b, int m, numtype * c);
int main ( int argc, char ** argv) {
- int n = LISTSIZE, m = LISTSIZE;
+ int n = LISTSIZEA, m = LISTSIZEB;
int p = 1;
int opt, i;
- int a_len, b_len, b_len_end, b_len_begin;
+ int block, b_len, b_len_begin, b_len_end;
numtype * c;
double startTime, endTime;
@@ -38,45 +39,62 @@ int main ( int argc, char ** argv) {
}
}
- //printf ("Number of processes that will be started: %i\n", p);
- //printf ("----------------------------------\n");
+ if (p > LISTSIZEA) {
+ printf ("Zu viele Aufteilungen!\n");
+ exit(1);
+ }
+ if (((int)(LISTSIZEA/p))*p != LISTSIZEA) {
+ printf ("Liste ist nicht teilbar durch %i\n", p);
+ exit(1);
+ }
+
+#ifdef DEBUG
+ printf ("Number of processes that will be started: %i\n", p);
+ printf ("----------------------------------\n");
+#endif
- c = (numtype *) malloc((LISTSIZE * 2 + 1) * (sizeof (numtype)));
- c[LISTSIZE*2] = -1;
+ c = (numtype *) malloc((LISTSIZEA+LISTSIZEB) * (sizeof (numtype)));
+ for (i = 0; i < (LISTSIZEA+LISTSIZEB); i++) {
+ c[i] = 0;
+ }
- //printlist("0 Sorted List A:", a);
- //printlist("0 Sorted List B:", b);
+#ifdef DEBUG
+ printlist("0 Sorted List A:", a, LISTSIZEA);
+ printlist("0 Sorted List B:", b, LISTSIZEB);
+#endif
startTime = omp_get_wtime();
- a_len = n/p;
- #pragma omp parallel for shared(a,b,c,n,m,p,a_len) private(i,b_len_begin,b_len_end,b_len)
+ block = n/p;
+ //#pragma omp parallel for shared(a,b,c,n,m,p,block) private(i,b_len,b_len_begin,b_len_end)
for (i = 0; i < p; i++) {
- b_len_begin = rank(a[i*a_len], b, m);
- b_len_end = rank(a[(i+1)*a_len], b, m);
- if (b_len_begin < 0) {
- printf ("Insert to end of list!\n");
- b_len_begin = n;
- }
- if (b_len_end < 0) {
- //printf ("Reached end of list!\n");
- b_len_end = n;
- }
+ b_len_begin = rank(a[i*block],b,m);
+ b_len_end = rank(a[(i+1)*block],b,m);
b_len = b_len_end - b_len_begin;
- //printf ("%i a_len: %i, b_len: %i (begin:%i ([%i]) -> end:%i [%i])\n", i+1, a_len, b_len, b_len_begin, b[b_len_begin], b_len_end, b[b_len_end]);
- merge( i+1,
- &a[i*a_len],
- a_len,
+ if (i == p-1) { b_len = m-b_len_begin; }
+#ifdef DEBUG
+ //printf ("b_len_begin: %i, b_len_end: %i, b_len: %i\n", b_len_begin, b_len_end, b_len);
+#endif
+
+ merge(i+1,
+ &a[i*block],
+ block,
&b[b_len_begin],
b_len,
- &c[i*a_len+b_len_begin]);
+ &c[i*block+b_len_begin]
+ );
}
endTime = omp_get_wtime();
printf("took %f seconds.\n", endTime-startTime);
- //printlist("Sorted List:", c);
- free(c);
+#ifdef DEBUG
+ printlist("Sorted List:", c, LISTSIZEA+LISTSIZEB);
+ printf ("------------\n");
+ merge(9, a, n, b, m, c);
+ printlist("Should Be: ", c, LISTSIZEA+LISTSIZEB);
+#endif
+ //free(c);
return 0;
}
@@ -89,14 +107,18 @@ void printlist(char * message, numtype * ptr, int len) {
printf ("\n");
}
+/*
+ * return -1 if len == 0
+ * return pos if elem is to be placed before *list
+ * return len if elem has to be positioned after list
+ */
int rank(numtype elem, numtype * list, int len) {
int pos;
int i = len;
-
pos = 0;
- if (elem == -1) { return -1; }
+ if (len == 0) { return -1; }
while (i > 0) {
- /*printf ("elem_list: %i %i\n", elem, *list);*/
+ /*printf ("rank: %i %i\n", elem, *list);*/
if (elem <= *list) {
return pos;
}
@@ -110,11 +132,17 @@ int rank(numtype elem, numtype * list, int len) {
void merge(int ti, numtype * a, int n, numtype * b, int m, int * c) {
int sum;
int i;
- /*printf ("sorting a:%i (%i) and b:%i (%i)\n", *a, n, *b, m);*/
+#ifdef DEBUG
+ //printf ("sorting a:%i (%i) and b:%i (%i)\n", *a, n, *b, m);
+ //printlist("sortinglista", a, n);
+ //printlist("sortinglistb", b, m);
+#endif
if (m<0) { m=0;}
if (n<0) { n=0;}
sum = n + m;
+#ifdef DEBUG
//printf ("%i modifying %i (%i+%i) (c[%i]/0x%08x -> c[%i]/0x%08x)\n", ti, sum, n, m, 0, (unsigned int) &c, sum-1, ((unsigned int) &c)+sum);
+#endif
for (i = 0; i < sum; i++) {
// n+m == 0
if (n <= 0 && m <= 0) {
@@ -122,19 +150,25 @@ void merge(int ti, numtype * a, int n, numtype * b, int m, int * c) {
return;
}
if (n <= 0) {
+ //printf ("changing: c[%i] from %i to %i\n", i, c[i], *b);
c[i] = *b++; m--;
} else {
if (m <= 0) {
+ //printf ("changing: c[%i] from %i to %i\n", i, c[i], *a);
c[i] = *a++; n--;
} else {
if (*a < *b) {
+ //printf ("changing: c[%i] from %i to %i\n", i, c[i], *a);
c[i] = *a++; n--;
} else {
+ //printf ("changing: c[%i] from %i to %i\n", i, c[i], *b);
c[i] = *b++; m--;
}
}
}
}
- // printf ("merge done, n=%d, m=%d\n", n, m);
+#ifdef DEBUG
+ //printf ("merge done, n=%d, m=%d\n", n, m);
+#endif
return;
}
--
2.43.0
From 16075ee2679344e8e32bb47e6471d1e3c7f507ba Mon Sep 17 00:00:00 2001
From: David Kaufmann <astra@ionic.at>
Date: Mon, 30 Jan 2012 15:17:51 +0100
Subject: [PATCH 09/16] re-add openmp
---
merge/sort.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/merge/sort.c b/merge/sort.c
index aae6c4a..2146652 100644
--- a/merge/sort.c
+++ b/merge/sort.c
@@ -66,7 +66,7 @@ int main ( int argc, char ** argv) {
startTime = omp_get_wtime();
block = n/p;
- //#pragma omp parallel for shared(a,b,c,n,m,p,block) private(i,b_len,b_len_begin,b_len_end)
+ #pragma omp parallel for shared(a,b,c,n,m,p,block) private(i,b_len,b_len_begin,b_len_end)
for (i = 0; i < p; i++) {
b_len_begin = rank(a[i*block],b,m);
b_len_end = rank(a[(i+1)*block],b,m);
--
2.43.0
From bc73ab91e781549e3600c5f33cf5974224429931 Mon Sep 17 00:00:00 2001
From: David Kaufmann <astra@ionic.at>
Date: Mon, 30 Jan 2012 15:34:03 +0100
Subject: [PATCH 10/16] update testscript
---
merge/generate_random.sh | 6 +++---
merge/test_threads.sh | 18 ++++++++++++------
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/merge/generate_random.sh b/merge/generate_random.sh
index 2554aa4..6a84d25 100755
--- a/merge/generate_random.sh
+++ b/merge/generate_random.sh
@@ -8,7 +8,7 @@ function pwait() {
}
# 10 ^ x values
-X=4
+X=$1
NULLEN=""
PUNKTE="."
for i in $(seq `expr $X - 1`);
@@ -53,8 +53,8 @@ rm -f unsorted1 unsorted2
rm -f numlist.h
echo "#define LISTSIZEA ${VALUES}" >> numlist.h
echo "#define LISTSIZEB ${VALUES}" >> numlist.h
-echo "extern int a["`expr ${VALUES} + 1`"];" >> numlist.h
-echo "extern int b["`expr ${VALUES} + 1`"];" >> numlist.h
+echo "extern int a[${VALUES}];" >> numlist.h
+echo "extern int b[${VALUES}];" >> numlist.h
rm -f numlist.c
cat numlist1.c >> numlist.c
diff --git a/merge/test_threads.sh b/merge/test_threads.sh
index 968a7e6..9bc3012 100755
--- a/merge/test_threads.sh
+++ b/merge/test_threads.sh
@@ -1,13 +1,19 @@
#!/bin/bash
-STATFILE=merge.dat
+STATFILE=merge
-rm $STATFILE
-for i in 1 2 4 5 8 10 20 25 40 50 100 125 200
+rm $STATFILE.*.dat
+for num in 3 4 5 6 7
do
- SORTMSG=`./sort -t $i`
- echo "$i:" $SORTMSG
- echo $SORTMSG | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i},\1/" >> $STATFILE
+ ./generate_random.sh $num
+ make clean
+ make
+ for i in 1 2 4 5 10 20 25 50 100
+ do
+ SORTMSG=`./sort -t $i`
+ echo "$i:" $SORTMSG
+ echo $SORTMSG | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i},\1/" >> $STATFILE.$num.dat
+ done
done
if [ $DISPLAY ]; then
--
2.43.0
From 69ad07d60d4ebbdebca8afd9e312ebb1e559c4c8 Mon Sep 17 00:00:00 2001
From: David Kaufmann <astra@ionic.at>
Date: Mon, 30 Jan 2012 15:56:07 +0100
Subject: [PATCH 11/16] graph generator added
---
merge/parseDat.py | 53 +++++++++++++++++++++++++++++++++++++++++++
merge/test_threads.sh | 16 +++++++------
2 files changed, 62 insertions(+), 7 deletions(-)
create mode 100755 merge/parseDat.py
diff --git a/merge/parseDat.py b/merge/parseDat.py
new file mode 100755
index 0000000..9570440
--- /dev/null
+++ b/merge/parseDat.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python2
+
+import os, subprocess
+
+def listtofile(listing, file):
+ fd = open(file, 'r')
+ for i in listing:
+ fd.write(i+"\n")
+ fd.close()
+
+def makeGraphs(graphs, graphtitle = "Title", xlabel = "Size of array", ylabel = "Time needed for calculation"):
+ gendir = "stats/generated/"
+ subprocess.call(["mkdir", "-p", gendir])
+ p = subprocess.Popen(['gnuplot'],stdout=subprocess.PIPE,stdin=subprocess.PIPE)
+ # now do the plots
+ for graphfile in graphs:
+ # set output 'filename'
+ p.stdin.write("set terminal png font arial 8 #size 600,300\n")
+ p.stdin.write("set grid\n")
+ p.stdin.write("set datafile separator ';'\n")
+ p.stdin.write("set title 'Timing of each "+graphtitle+"'\n")
+ p.stdin.write("set xlabel '"+xlabel+"'\n")
+ p.stdin.write("set ylabel '"+ylabel+"'\n")
+ p.stdin.write("set output '"+gendir+graphfile+"'\n")
+ # plot 'filename' using 1:2 title 'algorithm' with lines, 'filename2' using 1:2 title 'algo2' with lines
+ plots = []
+ for title, file, cols in graphs[graphfile]:
+ plots.append("'"+file+"' using "+cols+" title '"+title+"' with lines")
+ plotcommand = ", ".join(plots)
+ #print gendir+graphfile
+ p.stdin.write("plot "+plotcommand+"\n")
+ p.communicate()[0]
+ p.stdin.close()
+
+statsdir = 'stats/raw/'
+runs = os.listdir(statsdir)
+#print runs
+
+gendir = "stats/generated/"
+graphs = []
+outpng = 'merge.png'
+subprocess.call(["mkdir", "-p", gendir])
+for i in runs:
+ runfiles = os.listdir(statsdir+i)
+# print runfiles
+ for file in runfiles:
+# print "File: %s%s/%s" % (statsdir, i, file)
+ splitfilename = file.split('.')
+ dict = {}
+ dict['arrsize'] = splitfilename[1]
+ graphs.append((splitfilename[1], statsdir+i+'/'+file, '1:2'))
+
+makeGraphs({outpng: graphs}, "merge")
diff --git a/merge/test_threads.sh b/merge/test_threads.sh
index 9bc3012..85c9792 100755
--- a/merge/test_threads.sh
+++ b/merge/test_threads.sh
@@ -2,20 +2,22 @@
STATFILE=merge
-rm $STATFILE.*.dat
-for num in 3 4 5 6 7
+STARTTS="`date --iso-8601=minutes`"
+mkdir -p stats/raw/${STARTTS}
+EXPO="3 4 5 6 7"
+PARTS="1 2 4 5 10 20 25 50 100"
+for num in $EXPO
do
./generate_random.sh $num
make clean
make
- for i in 1 2 4 5 10 20 25 50 100
+ for i in $PARTS
do
- SORTMSG=`./sort -t $i`
- echo "$i:" $SORTMSG
- echo $SORTMSG | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i},\1/" >> $STATFILE.$num.dat
+ foo=`expr 10**$num`
+ ./sort -t $i | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i};\1/" | tee -a stats/raw/$STARTTS/$STATFILE.$foo.dat
done
done
if [ $DISPLAY ]; then
- gnuplot merge.plt && display merge.png
+ ./parseDat.py
fi
--
2.43.0
From e4ddb3f489a6bbdafdd963673dddacfacf7f53c9 Mon Sep 17 00:00:00 2001
From: David Kaufmann <astra@ionic.at>
Date: Mon, 30 Jan 2012 16:04:25 +0100
Subject: [PATCH 12/16] fix filenames
---
merge/test_threads.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/merge/test_threads.sh b/merge/test_threads.sh
index 85c9792..7848c5d 100755
--- a/merge/test_threads.sh
+++ b/merge/test_threads.sh
@@ -13,7 +13,7 @@ do
make
for i in $PARTS
do
- foo=`expr 10**$num`
+ let foo=`expr 10**$num`
./sort -t $i | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i};\1/" | tee -a stats/raw/$STARTTS/$STATFILE.$foo.dat
done
done
--
2.43.0
From d4da095b12473992821a06fde57b0bd2ac05ec54 Mon Sep 17 00:00:00 2001
From: David Kaufmann <astra@ionic.at>
Date: Mon, 30 Jan 2012 16:24:32 +0100
Subject: [PATCH 13/16] optimize merge
---
merge/parseDat.py | 2 +-
merge/test_threads.sh | 14 ++++++++++----
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/merge/parseDat.py b/merge/parseDat.py
index 9570440..fa97ffa 100755
--- a/merge/parseDat.py
+++ b/merge/parseDat.py
@@ -8,7 +8,7 @@ def listtofile(listing, file):
fd.write(i+"\n")
fd.close()
-def makeGraphs(graphs, graphtitle = "Title", xlabel = "Size of array", ylabel = "Time needed for calculation"):
+def makeGraphs(graphs, graphtitle = "Title", xlabel = "Number of Parts", ylabel = "Time needed for calculation"):
gendir = "stats/generated/"
subprocess.call(["mkdir", "-p", gendir])
p = subprocess.Popen(['gnuplot'],stdout=subprocess.PIPE,stdin=subprocess.PIPE)
diff --git a/merge/test_threads.sh b/merge/test_threads.sh
index 7848c5d..fce8f99 100755
--- a/merge/test_threads.sh
+++ b/merge/test_threads.sh
@@ -4,17 +4,23 @@ STATFILE=merge
STARTTS="`date --iso-8601=minutes`"
mkdir -p stats/raw/${STARTTS}
-EXPO="3 4 5 6 7"
+EXPO=$(seq 1000 1000 10000) $(seq 10000 10000 100000)
PARTS="1 2 4 5 10 20 25 50 100"
+let maxfoo=`expr 10**8`
for num in $EXPO
do
- ./generate_random.sh $num
+ echo ""
+ rm -f numlist.h
+ echo "#define LISTSIZEA ${num}" >> numlist.h
+ echo "#define LISTSIZEB ${num}" >> numlist.h
+ echo "extern int a[${maxfoo}];" >> numlist.h
+ echo "extern int b[${maxfoo}];" >> numlist.h
+
make clean
make
for i in $PARTS
do
- let foo=`expr 10**$num`
- ./sort -t $i | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i};\1/" | tee -a stats/raw/$STARTTS/$STATFILE.$foo.dat
+ ./sort -t $i | sed "s/took \([0-9]*\.[0-9]*\) seconds\./${i};\1/" | tee -a stats/raw/$STARTTS/$STATFILE.$num.dat
done
done
--
2.43.0
From 07776a394abfbae9952808c574cdf5284e686182 Mon Sep 17 00:00:00 2001
From: David Kaufmann <astra@ionic.at>
Date: Mon, 30 Jan 2012 16:46:50 +0100
Subject: [PATCH 14/16] fix merge
---
merge/test_threads.sh | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/merge/test_threads.sh b/merge/test_threads.sh
index fce8f99..fd7db17 100755
--- a/merge/test_threads.sh
+++ b/merge/test_threads.sh
@@ -2,9 +2,13 @@
STATFILE=merge
+#
+# first run ./generate_random 8
+#
+
STARTTS="`date --iso-8601=minutes`"
mkdir -p stats/raw/${STARTTS}
-EXPO=$(seq 1000 1000 10000) $(seq 10000 10000 100000)
+EXPO="$(seq 1000 1000 10000) $(seq 10000 10000 100000)"
PARTS="1 2 4 5 10 20 25 50 100"
let maxfoo=`expr 10**8`
for num in $EXPO
@@ -16,7 +20,6 @@ do
echo "extern int a[${maxfoo}];" >> numlist.h
echo "extern int b[${maxfoo}];" >> numlist.h
- make clean
make
for i in $PARTS
do
--
2.43.0
From a01422e1bdbf78f2fc5e10a0d6ced9e4824bc228 Mon Sep 17 00:00:00 2001
From: Jan Vales <e0726236@jupiter.par.tuwien.ac.at>
Date: Mon, 30 Jan 2012 18:12:48 +0100
Subject: [PATCH 15/16] i can haz graphs plz?
---
scan/runAll.sh | 24 +++++++++++++++++++++++-
scan/scan.c | 12 +++++++-----
2 files changed, 30 insertions(+), 6 deletions(-)
diff --git a/scan/runAll.sh b/scan/runAll.sh
index 7de8905..c57462b 100755
--- a/scan/runAll.sh
+++ b/scan/runAll.sh
@@ -11,9 +11,31 @@ if [ ! -e build ]; then
fi
STARTTS="`date --iso-8601=minutes`"
+NODELIST="$(seq 0 4 32)"
+NNPLIST="$(seq 0 4 16)"
+RUNLIST="$(seq 0 25000 100000) $(seq 0 250000 1000000) $(seq 0 2500000 10000000)"
echo "STARTING: stats/raw/${STARTTS}"
mkdir -p "stats/raw/${STARTTS}"
-mpirun -node 1-32 -nnp 1 build/scan &> stats/raw/tmp && echo "done" && mv stats/raw/tmp "stats/raw/${STARTTS}/scan"
+for nno in $NODELIST; do
+ for nnnp in $NNPLIST; do
+ for i in $RUNLIST; do
+ if [ "$nno" -eq "0" ] ; then
+ nno=2
+ fi
+ if [ "$nnnp" -eq "0" ] ; then
+ nnnp=1
+ fi
+ if [ "$i" -eq "0" ] ; then
+ echo "* ignoring 0 run *"
+ else
+ echo "running with nno:${nno} nnnp:${nnnp} n:${i}"
+ mpirun -node 1-${nno} -nnp ${nnnp} build/scan -n${i} &> stats/tmp && mv stats/tmp "stats/raw/${STARTTS}/no${nno}_nnp${nnnp}_n${i}"
+ cat "stats/raw/${STARTTS}/no${nno}_nnp${nnnp}_n${i}" | grep "timing"
+ fi
+ done
+ done
+done
+
diff --git a/scan/scan.c b/scan/scan.c
index aff36b0..0f5badd 100755
--- a/scan/scan.c
+++ b/scan/scan.c
@@ -169,23 +169,22 @@ int main(int argc, char *argv[]){
MPI_Finalize();
exit(3);
}
-
+#ifdef DEBUG
fprintf(stdout, "[%d/%d:%s] file read - distributing work.\n",rank,nodes,name);
+#endif
startTime = MPI_Wtime();
MPI_Isend(databuf, size-1, MPI_LONG, 0, KEY, MPI_COMM_WORLD, &request);
}
- /* stuff done by all nodes */
+ /* receive work and propagate to next node, if not last node. */
if (startTime == 0) startTime = MPI_Wtime();
tmp = rank -1;
if(tmp < 0) tmp = 0;
MPI_Recv(databuf, size, MPI_LONG, tmp, KEY, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
-
- /* propagate data, if not last node. */
if(rank +1 < nodes) MPI_Send(databuf, size, MPI_LONG, rank+1, KEY, MPI_COMM_WORLD);
prepTime = MPI_Wtime();
- /* do processing here. */
+ /* do actual work here. */
for(unsigned long i = (rank*blocksize+1); i < ((rank+1)*blocksize) && i < size; i++){
databuf[i] = databuf[(i-1)] + databuf[i];
}
@@ -224,14 +223,17 @@ int main(int argc, char *argv[]){
MPI_Recv(databuf, size, MPI_LONG, nodes-1, KEY, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
endTime = MPI_Wtime();
fprintf(stdout, "[%d/%d:%s] result: %li\n",rank,nodes,name,databuf[size-1]);
+ fprintf(stdout, "[%d/%d:%s] timings: prep:%f algo:%f postp:%f end:%f\n",rank,nodes,name,prepTime-startTime,algoTime-startTime,postpTime-startTime,endTime-startTime);
#ifdef DEBUG
fprintf(stdout, "[%d/%d:%s] res array ",rank,nodes,name);
array_contents(databuf, size);
#endif
}
+#ifdef DEBUG
if(endTime == 0) endTime = MPI_Wtime();
fprintf(stdout, "[%d/%d:%s] timings: prep:%f algo:%f postp:%f end:%f\n",rank,nodes,name,prepTime-startTime,algoTime-startTime,postpTime-startTime,endTime-startTime);
+#endif
MPI_Finalize();
return 0;
}
--
2.43.0
From 7b8a85ea46bf4709ca3522e3f87e67c784016986 Mon Sep 17 00:00:00 2001
From: Jan Vales <e0726236@jupiter.par.tuwien.ac.at>
Date: Mon, 30 Jan 2012 18:38:50 +0100
Subject: [PATCH 16/16] fix'd deadlock if there is just 1 node.
---
scan/runAll.sh | 2 +-
scan/scan.c | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/scan/runAll.sh b/scan/runAll.sh
index c57462b..dabdbb1 100755
--- a/scan/runAll.sh
+++ b/scan/runAll.sh
@@ -22,7 +22,7 @@ for nno in $NODELIST; do
for nnnp in $NNPLIST; do
for i in $RUNLIST; do
if [ "$nno" -eq "0" ] ; then
- nno=2
+ nno=1
fi
if [ "$nnnp" -eq "0" ] ; then
nnnp=1
diff --git a/scan/scan.c b/scan/scan.c
index 0f5badd..35446c1 100755
--- a/scan/scan.c
+++ b/scan/scan.c
@@ -48,6 +48,7 @@ int main(int argc, char *argv[]){
int tmp2;
FILE *file = NULL;
MPI_Request request;
+ MPI_Status status;
unsigned long blocksize;
unsigned long *databuf;
@@ -216,7 +217,7 @@ int main(int argc, char *argv[]){
if(tmp >= nodes) tmp = 0;
tmp2 = ((rank+1)*blocksize);
if(tmp2 > size) tmp2 = size;
- MPI_Send(databuf, tmp2, MPI_LONG, tmp, KEY, MPI_COMM_WORLD);
+ MPI_Isend(databuf, tmp2, MPI_LONG, tmp, KEY, MPI_COMM_WORLD,&request);
/* receive result by root */
if(rank == 0){
@@ -234,6 +235,7 @@ int main(int argc, char *argv[]){
if(endTime == 0) endTime = MPI_Wtime();
fprintf(stdout, "[%d/%d:%s] timings: prep:%f algo:%f postp:%f end:%f\n",rank,nodes,name,prepTime-startTime,algoTime-startTime,postpTime-startTime,endTime-startTime);
#endif
+ MPI_Wait(&request, &status);
MPI_Finalize();
return 0;
}
--
2.43.0