From 2e553811981364419915e50d2e2c8ff80185e37f Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 4 Nov 2015 10:08:33 +0100 Subject: [PATCH 01/74] Started working on ddcd --- ddcd.c | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 ddcd.c diff --git a/ddcd.c b/ddcd.c new file mode 100644 index 0000000..f93d1ce --- /dev/null +++ b/ddcd.c @@ -0,0 +1,96 @@ +/* +This software is part of libcsdr, a set of simple DSP routines for +Software Defined Radio. + +Copyright (c) 2014, Andras Retzler +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL ANDRAS RETZLER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include +#include + + + +struct client_s +{ + int socket; + int addr; +} client_t; + + + +int main(int argc, char* argv[]) +{ + int c; + + //arguments: + int host_port; + char host_address[100] = "127.0.0.1"; + int decimation; + + for(;;) + { + int option_index = 0; + static struct option long_options[] = { + {"port", required_argument, 0, 'p' }, + {"address", required_argument, 0, 'a' }, + {"decimation", required_argument, 0, 'd' } + }; + c = getopt_long(argc, argv, "p:a:d:", long_options, &option_index); + if(c==-1) break; + switch (c) + { + case 'a': + host_address[100-1]=0; + strncpy(host_address,optarg,100-1); + break; + case 'p': + host_port=atoi(optarg); + break; + case 'd': + decimation=atoi(optarg); + break; + case 0: + case '?': + default: + printf(" 0%o ??\n", c); + } + + + + + } + + struct sockaddr_in addr_host; + int listen_socket; + std::vector clients; + listen_socket=socket(AF_INET,SOCK_STREAM,0); + memset(&addr_host,'0',sizeof(addr_host)); + addr_host.sin_family=AF_INET; + addr_host.sin_port=htons(8888); + addr_host.sin_addr.s_addr=inet_addr("127.0.0.1"); + +} From 7859ac9d397cdab40c5f9cc4cb63f42c04798750 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 4 Nov 2015 10:31:54 +0100 Subject: [PATCH 02/74] getopt() now works in ddcd. Also fixed Makefile for only compiling those targets that have their corresponding sources changed. --- Makefile | 10 ++++++++-- ddcd.c | 26 +++++++++++++++----------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 4bcd3f8..818f1fc 100644 --- a/Makefile +++ b/Makefile @@ -43,20 +43,26 @@ PARAMS_SO = -fpic PARAMS_MISC = -Wno-unused-result FFTW_PACKAGE = fftw-3.3.3 -all: clean-vect +.PHONY: clean-vect clean +all: csdr ddcd +libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c @echo NOTE: you may have to manually edit Makefile to optimize for your CPU \(especially if you compile on ARM, please edit PARAMS_NEON\). @echo Auto-detected optimization parameters: $(PARAMS_SIMD) @echo + rm -f dumpvect*.vect gcc -std=gnu99 $(PARAMS_LOOPVECT) $(PARAMS_SIMD) $(LIBSOURCES) $(PARAMS_LIBS) $(PARAMS_MISC) -fpic -shared -o libcsdr.so -./parsevect dumpvect*.vect +csdr: csdr.c libcsdr.so gcc -std=gnu99 $(PARAMS_LOOPVECT) $(PARAMS_SIMD) csdr.c $(PARAMS_LIBS) -L. -lcsdr $(PARAMS_MISC) -o csdr +ddcd: ddcd.c libcsdr.so + gcc -std=gnu99 $(PARAMS_LOOPVECT) $(PARAMS_SIMD) ddcd.c $(PARAMS_LIBS) -L. -lcsdr $(PARAMS_MISC) -o ddcd arm-cross: clean-vect #note: this doesn't work since having added FFTW arm-linux-gnueabihf-gcc -std=gnu99 -O3 -fshort-double -ffast-math -dumpbase dumpvect-arm -fdump-tree-vect-details -mfloat-abi=softfp -march=armv7-a -mtune=cortex-a9 -mfpu=neon -mvectorize-with-neon-quad -Wno-unused-result -Wformat=0 $(SOURCES) -lm -o ./csdr clean-vect: rm -f dumpvect*.vect clean: clean-vect - rm -f libcsdr.so csdr + rm -f libcsdr.so csdr ddcd install: install -m 0755 libcsdr.so /usr/lib install -m 0755 csdr /usr/bin diff --git a/ddcd.c b/ddcd.c index f93d1ce..5dd0bd1 100644 --- a/ddcd.c +++ b/ddcd.c @@ -31,8 +31,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include +#include - +#define SOFTWARE_NAME "ddcd" +#define MSG_START SOFTWARE_NAME ": " struct client_s { @@ -40,16 +43,16 @@ struct client_s int addr; } client_t; - +int host_port = 0; +char host_address[100] = "127.0.0.1"; +int decimation = 0; int main(int argc, char* argv[]) { int c; //arguments: - int host_port; - char host_address[100] = "127.0.0.1"; - int decimation; + for(;;) { @@ -75,16 +78,16 @@ int main(int argc, char* argv[]) break; case 0: case '?': + case ':': default: printf(" 0%o ??\n", c); } - - - - } - - struct sockaddr_in addr_host; + + if(!decimation) { fprintf(stderr, MSG_START "missing required command line argument, --decimation.\n"); exit(1); } + if(!host_port) { fprintf(stderr, MSG_START "missing required command line argument, --port.\n"); exit(1); } + + /*struct sockaddr_in addr_host; int listen_socket; std::vector clients; listen_socket=socket(AF_INET,SOCK_STREAM,0); @@ -92,5 +95,6 @@ int main(int argc, char* argv[]) addr_host.sin_family=AF_INET; addr_host.sin_port=htons(8888); addr_host.sin_addr.s_addr=inet_addr("127.0.0.1"); + */ } From 2b8101ddf6e87483d812f6ed29318bf3b6e60fa3 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 4 Nov 2015 13:31:47 +0100 Subject: [PATCH 03/74] Working on the socket server now. --- Makefile | 6 ++-- ddcd.c => ddcd.cpp | 72 +++++++++++++++++++++++++++++++++++++++------- ddcd.h | 1 + 3 files changed, 65 insertions(+), 14 deletions(-) rename ddcd.c => ddcd.cpp (66%) create mode 100644 ddcd.h diff --git a/Makefile b/Makefile index 818f1fc..5de63bd 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ FFTW_PACKAGE = fftw-3.3.3 .PHONY: clean-vect clean all: csdr ddcd -libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c +libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c *.h @echo NOTE: you may have to manually edit Makefile to optimize for your CPU \(especially if you compile on ARM, please edit PARAMS_NEON\). @echo Auto-detected optimization parameters: $(PARAMS_SIMD) @echo @@ -54,8 +54,8 @@ libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c -./parsevect dumpvect*.vect csdr: csdr.c libcsdr.so gcc -std=gnu99 $(PARAMS_LOOPVECT) $(PARAMS_SIMD) csdr.c $(PARAMS_LIBS) -L. -lcsdr $(PARAMS_MISC) -o csdr -ddcd: ddcd.c libcsdr.so - gcc -std=gnu99 $(PARAMS_LOOPVECT) $(PARAMS_SIMD) ddcd.c $(PARAMS_LIBS) -L. -lcsdr $(PARAMS_MISC) -o ddcd +ddcd: ddcd.cpp libcsdr.so ddcd.h + g++ $(PARAMS_LOOPVECT) $(PARAMS_SIMD) ddcd.cpp $(PARAMS_LIBS) -L. -lcsdr $(PARAMS_MISC) -o ddcd arm-cross: clean-vect #note: this doesn't work since having added FFTW arm-linux-gnueabihf-gcc -std=gnu99 -O3 -fshort-double -ffast-math -dumpbase dumpvect-arm -fdump-tree-vect-details -mfloat-abi=softfp -march=armv7-a -mtune=cortex-a9 -mfpu=neon -mvectorize-with-neon-quad -Wno-unused-result -Wformat=0 $(SOURCES) -lm -o ./csdr diff --git a/ddcd.c b/ddcd.cpp similarity index 66% rename from ddcd.c rename to ddcd.cpp index 5dd0bd1..d6d9403 100644 --- a/ddcd.c +++ b/ddcd.cpp @@ -28,19 +28,28 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include "ddcd.h" #include #include #include #include #include +#include +#include +#include +#include +#include +#include #define SOFTWARE_NAME "ddcd" #define MSG_START SOFTWARE_NAME ": " -struct client_s +typedef struct client_s { + struct sockaddr_in addr; int socket; - int addr; + pid_t pid; + int pipefd[2]; } client_t; int host_port = 0; @@ -51,9 +60,6 @@ int main(int argc, char* argv[]) { int c; - //arguments: - - for(;;) { int option_index = 0; @@ -87,14 +93,58 @@ int main(int argc, char* argv[]) if(!decimation) { fprintf(stderr, MSG_START "missing required command line argument, --decimation.\n"); exit(1); } if(!host_port) { fprintf(stderr, MSG_START "missing required command line argument, --port.\n"); exit(1); } - /*struct sockaddr_in addr_host; + struct sockaddr_in addr_host; int listen_socket; - std::vector clients; + std::vector clients(10); listen_socket=socket(AF_INET,SOCK_STREAM,0); memset(&addr_host,'0',sizeof(addr_host)); addr_host.sin_family=AF_INET; - addr_host.sin_port=htons(8888); - addr_host.sin_addr.s_addr=inet_addr("127.0.0.1"); - */ - + addr_host.sin_port=htons(host_port); + + if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE) + { fprintf(stderr, MSG_START "invalid host address.\n"); exit(1); } + + if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0) + { fprintf(stderr, MSG_START "cannot bind() address to the socket.\n"); exit(1); } + + if( listen(listen_socket, 10) == -1) + { fprintf(stderr, MSG_START "cannot listen() on socket.\n"); exit(1); } + + for(;;) + { + struct sockaddr_in addr_cli; + socklen_t addr_cli_len; + int new_socket; + + if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) == -1) + { + fprintf(stderr, MSG_START "cannot accept() a connection.\n"); + continue; + } + + client_t* new_client = new client_t; + memcpy(&new_client->addr, &addr_cli, sizeof(new_client->addr)); + new_client->socket = new_socket; + + if(new_client->pid = fork()) + { + //We're the parent + clients.push_back(new_client); + printf("client pid: %d\n", new_client->pid); + } + else + { + //We're the client + client(); + break; + } + } + + return 0; +} + +void client() +{ + printf("I'm the client\n"); + for(;;) sleep(1); } diff --git a/ddcd.h b/ddcd.h new file mode 100644 index 0000000..02ed196 --- /dev/null +++ b/ddcd.h @@ -0,0 +1 @@ +void client(); From e694e5a5b38fdce6ea131ea73d561a33af68e727 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 4 Nov 2015 18:11:12 +0100 Subject: [PATCH 04/74] Solved the "cannot accept connection" problem. --- ddcd.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index d6d9403..d40878b 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -100,22 +100,23 @@ int main(int argc, char* argv[]) memset(&addr_host,'0',sizeof(addr_host)); addr_host.sin_family=AF_INET; addr_host.sin_port=htons(host_port); + addr_host.sin_addr.s_addr = INADDR_ANY; - if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE) + if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) { fprintf(stderr, MSG_START "invalid host address.\n"); exit(1); } - if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0) + if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) { fprintf(stderr, MSG_START "cannot bind() address to the socket.\n"); exit(1); } - if( listen(listen_socket, 10) == -1) + if( listen(listen_socket, 10) == -1 ) { fprintf(stderr, MSG_START "cannot listen() on socket.\n"); exit(1); } for(;;) { struct sockaddr_in addr_cli; - socklen_t addr_cli_len; + socklen_t addr_cli_len = sizeof(addr_cli); int new_socket; - + if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) == -1) { fprintf(stderr, MSG_START "cannot accept() a connection.\n"); From 37555f2a777344c9890345a829d13d70af3a64e7 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 5 Nov 2015 23:57:03 +0100 Subject: [PATCH 05/74] This is a kind of stream multiplexer now (stdin input, TCP output to multiple clients), but closing clients is not handled yet. --- ddcd.cpp | 158 ++++++++++++++++++++++++++++++++++++++----------------- ddcd.h | 27 ++++++++++ 2 files changed, 138 insertions(+), 47 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index d40878b..3e6202a 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -29,36 +29,33 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "ddcd.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include + #define SOFTWARE_NAME "ddcd" #define MSG_START SOFTWARE_NAME ": " -typedef struct client_s -{ - struct sockaddr_in addr; - int socket; - pid_t pid; - int pipefd[2]; -} client_t; - int host_port = 0; char host_address[100] = "127.0.0.1"; int decimation = 0; +int bufsize = 1024; +int bufsizeall; +char* buf; + +int set_nonblocking(int fd) +{ + int flagtmp; + if((flagtmp = fcntl(fd, F_GETFL))!=-1) + if((flagtmp = fcntl(fd, F_SETFL, flagtmp|O_NONBLOCK))!=-1) + return 0; + return 1; +} + +client_t* this_client; int main(int argc, char* argv[]) { int c; + fd_set select_fds; for(;;) { @@ -66,9 +63,10 @@ int main(int argc, char* argv[]) static struct option long_options[] = { {"port", required_argument, 0, 'p' }, {"address", required_argument, 0, 'a' }, - {"decimation", required_argument, 0, 'd' } + {"decimation", required_argument, 0, 'd' }, + {"bufsize", required_argument, 0, 'b' } }; - c = getopt_long(argc, argv, "p:a:d:", long_options, &option_index); + c = getopt_long(argc, argv, "p:a:d:b:", long_options, &option_index); if(c==-1) break; switch (c) { @@ -82,6 +80,9 @@ int main(int argc, char* argv[]) case 'd': decimation=atoi(optarg); break; + case 'b': + bufsize=atoi(optarg); + break; case 0: case '?': case ':': @@ -90,62 +91,125 @@ int main(int argc, char* argv[]) } } - if(!decimation) { fprintf(stderr, MSG_START "missing required command line argument, --decimation.\n"); exit(1); } - if(!host_port) { fprintf(stderr, MSG_START "missing required command line argument, --port.\n"); exit(1); } + if(!decimation) error_exit(MSG_START "missing required command line argument, --decimation.\n"); + if(!host_port) error_exit(MSG_START "missing required command line argument, --port.\n"); struct sockaddr_in addr_host; int listen_socket; - std::vector clients(10); + std::vector clients; + clients.reserve(100); listen_socket=socket(AF_INET,SOCK_STREAM,0); + + int sockopt = 1; + if( setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&sockopt, sizeof(sockopt)) == -1 ) + error_exit(MSG_START "cannot set SO_REUSEADDR.\n"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 + memset(&addr_host,'0',sizeof(addr_host)); addr_host.sin_family=AF_INET; addr_host.sin_port=htons(host_port); addr_host.sin_addr.s_addr = INADDR_ANY; if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) - { fprintf(stderr, MSG_START "invalid host address.\n"); exit(1); } + error_exit(MSG_START "invalid host address.\n"); if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) - { fprintf(stderr, MSG_START "cannot bind() address to the socket.\n"); exit(1); } + error_exit(MSG_START "cannot bind() address to the socket.\n"); if( listen(listen_socket, 10) == -1 ) - { fprintf(stderr, MSG_START "cannot listen() on socket.\n"); exit(1); } + error_exit(MSG_START "cannot listen() on socket.\n"); + + struct sockaddr_in addr_cli; + socklen_t addr_cli_len = sizeof(addr_cli); + int new_socket; + + //The server will wait on these sockets later... + + + //Set stdin and listen_socket to non-blocking + if(set_nonblocking(STDIN_FILENO) || set_nonblocking(listen_socket)) + error_exit(MSG_START "cannot set_nonblocking().\n"); + + bufsizeall = bufsize*sizeof(char); + buf = (char*)malloc(bufsizeall); + + FD_ZERO(&select_fds); + FD_SET(listen_socket, &select_fds); + FD_SET(STDIN_FILENO, &select_fds); + int highfd = ((listen_socket>STDIN_FILENO)?listen_socket:STDIN_FILENO) + 1; for(;;) { - struct sockaddr_in addr_cli; - socklen_t addr_cli_len = sizeof(addr_cli); - int new_socket; - - if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) == -1) + //Let's wait until there is any new data to read, or any new connection! + select(highfd, &select_fds, NULL, NULL, NULL); + + //Is there a new client connection? + if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) { - fprintf(stderr, MSG_START "cannot accept() a connection.\n"); - continue; + this_client = new client_t; + memcpy(&this_client->addr, &addr_cli, sizeof(this_client->addr)); + this_client->socket = new_socket; + if(pipe(this_client->pipefd) == -1) + { + perror(MSG_START "cannot open new pipe() for the client.\n"); + continue; + } + if(this_client->pid = fork()) + { + //We're the parent + set_nonblocking(this_client->pipefd[1]); + clients.push_back(this_client); + printf("client pid: %d\n", this_client->pid); + } + else + { + //We're the client + client(); + return 1; + } } - client_t* new_client = new client_t; - memcpy(&new_client->addr, &addr_cli, sizeof(new_client->addr)); - new_client->socket = new_socket; - - if(new_client->pid = fork()) + int retval = read(STDIN_FILENO, buf, bufsizeall); + if(retval==0) { - //We're the parent - clients.push_back(new_client); - printf("client pid: %d\n", new_client->pid); + //end of input stream, close clients and exit } - else + else if(retval != -1) { - //We're the client - client(); - break; + for (int i=0;ipipefd[1], buf, retval)==-1) + print_client(clients[i], "lost buffer, failed to write pipe"); + } } + //TODO: at the end, server closes pipefd[1] for client + //close(this_client->pipefd[1]); } return 0; } +void print_client(client_t* client, const char* what) +{ + fprintf(stderr,MSG_START " (client %s:%d) %s\n", inet_ntoa(client->addr.sin_addr), client->addr.sin_port, what); +} + +void client_cleanup() +{ + close(this_client->pipefd[0]); +} + void client() { printf("I'm the client\n"); - for(;;) sleep(1); + for(;;) + { + read(this_client->pipefd[0],buf,bufsizeall); + send(this_client->socket,buf,bufsizeall,0); + } +} + +void error_exit(const char* why) +{ + perror(why); + exit(1); } diff --git a/ddcd.h b/ddcd.h index 02ed196..1106a00 100644 --- a/ddcd.h +++ b/ddcd.h @@ -1 +1,28 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct client_s +{ + struct sockaddr_in addr; + int socket; + pid_t pid; + int pipefd[2]; +} client_t; + + void client(); +void error_exit(const char* why); +void print_client(client_t* client, const char* what); From 7bae8b1ad896a98dba2af6f4787c219ea2b92673 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Fri, 6 Nov 2015 10:45:43 +0100 Subject: [PATCH 06/74] Will have to debug clients[i]->error and waitpid() --- ddcd.cpp | 53 ++++++++++++++++++++++++++++++++++++++++++++++++----- ddcd.h | 3 +++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index 3e6202a..858c878 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -39,6 +39,7 @@ char host_address[100] = "127.0.0.1"; int decimation = 0; int bufsize = 1024; int bufsizeall; +int pipe_max_size; char* buf; int set_nonblocking(int fd) @@ -137,6 +138,22 @@ int main(int argc, char* argv[]) FD_SET(STDIN_FILENO, &select_fds); int highfd = ((listen_socket>STDIN_FILENO)?listen_socket:STDIN_FILENO) + 1; + FILE* tempfile = fopen("/proc/sys/fs/pipe-max-size","r"); + if(!tempfile) + { + perror(MSG_START "cannot read /proc/sys/fs/pipe-max-size"); + } + else + { + char pipe_max_size_str[100]; + int tfread = fread(pipe_max_size_str, 1, 100, tempfile); + pipe_max_size_str[tfread]='\0'; + pipe_max_size = atoi(pipe_max_size_str); + //fprintf(stderr, MSG_START "note: pipe_max_size = %d\n", pipe_max_size); + //if(pipe_max_size>4096 && fcntl(STDIN_FILENO, F_SETPIPE_SZ, pipe_max_size)==-1) + // perror("failed to fcntl(STDIN_FILENO, F_SETPIPE_SZ, ...)"); + } + for(;;) { //Let's wait until there is any new data to read, or any new connection! @@ -146,6 +163,7 @@ int main(int argc, char* argv[]) if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) { this_client = new client_t; + this_client->error = 0; memcpy(&this_client->addr, &addr_cli, sizeof(this_client->addr)); this_client->socket = new_socket; if(pipe(this_client->pipefd) == -1) @@ -153,6 +171,8 @@ int main(int argc, char* argv[]) perror(MSG_START "cannot open new pipe() for the client.\n"); continue; } + if(fcntl(this_client->pipefd[1], F_SETPIPE_SZ, pipe_max_size) == -1) + perror("failed to F_SETPIPE_SZ for the client pipe!"); if(this_client->pid = fork()) { //We're the parent @@ -175,14 +195,33 @@ int main(int argc, char* argv[]) } else if(retval != -1) { - for (int i=0;ipipefd[1], buf, retval)==-1) - print_client(clients[i], "lost buffer, failed to write pipe"); + { + + if(!clients[i]->error) print_client(clients[i], "lost buffer, failed to write pipe"); + else clients[i]->error=1; + //fprintf(stderr, MSG_START "errno is %d\n", errno); //usually 11 + int wpstatus; + int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); + if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); + else if(wpresult == 0) + { + //Client exited! + print_client(clients[i], "closing client from main process."); + close(clients[i]->pipefd[1]); + close(clients[i]->socket); + delete clients[i]; + clients.erase(clients.begin()+i); + print_client(clients[i], "okay."); + } + } + else clients[i]->error=0; } } //TODO: at the end, server closes pipefd[1] for client - //close(this_client->pipefd[1]); + // } return 0; @@ -200,11 +239,15 @@ void client_cleanup() void client() { - printf("I'm the client\n"); + fprintf(stderr, "I'm the client\n"); for(;;) { read(this_client->pipefd[0],buf,bufsizeall); - send(this_client->socket,buf,bufsizeall,0); + if(send(this_client->socket,buf,bufsizeall,0)==-1) + { + print_client(this_client, "closing."); + exit(0); + } } } diff --git a/ddcd.h b/ddcd.h index 1106a00..be2a291 100644 --- a/ddcd.h +++ b/ddcd.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include typedef struct client_s { @@ -20,6 +22,7 @@ typedef struct client_s int socket; pid_t pid; int pipefd[2]; + int error; } client_t; From cb1b6ac8e22e48879d0e26b8e2912295725a2218 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Fri, 6 Nov 2015 14:14:02 +0100 Subject: [PATCH 07/74] Now we seem to have a proper server that correctly closes clients. --- ddcd.cpp | 38 +++++++++++++++++++++++++------------- ddcd.h | 1 + 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index 858c878..37242e5 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -51,6 +51,12 @@ int set_nonblocking(int fd) return 1; } +int proc_exists(pid_t pid) +{ + if(pid==0 || pid==1) return 1; + return kill(pid, 0) != -1; +} + client_t* this_client; int main(int argc, char* argv[]) @@ -119,13 +125,12 @@ int main(int argc, char* argv[]) if( listen(listen_socket, 10) == -1 ) error_exit(MSG_START "cannot listen() on socket.\n"); + fprintf(stderr,MSG_START "listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); + struct sockaddr_in addr_cli; socklen_t addr_cli_len = sizeof(addr_cli); int new_socket; - //The server will wait on these sockets later... - - //Set stdin and listen_socket to non-blocking if(set_nonblocking(STDIN_FILENO) || set_nonblocking(listen_socket)) error_exit(MSG_START "cannot set_nonblocking().\n"); @@ -200,13 +205,20 @@ int main(int argc, char* argv[]) if(write(clients[i]->pipefd[1], buf, retval)==-1) { - if(!clients[i]->error) print_client(clients[i], "lost buffer, failed to write pipe"); - else clients[i]->error=1; + if(!clients[i]->error) + { + print_client(clients[i], "lost buffer, failed to write pipe."); + clients[i]->error=1; + } //fprintf(stderr, MSG_START "errno is %d\n", errno); //usually 11 - int wpstatus; - int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); - if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); - else if(wpresult == 0) + //int wpstatus; + //int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); + //fprintf(stderr, MSG_START "pid is %d\n",clients[i]->pid); + //perror("somethings wrong"); + //if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); + //else if(wpresult == 0) + waitpid(clients[i]->pid, NULL, WNOHANG); + if(!proc_exists(clients[i]->pid)) { //Client exited! print_client(clients[i], "closing client from main process."); @@ -214,10 +226,10 @@ int main(int argc, char* argv[]) close(clients[i]->socket); delete clients[i]; clients.erase(clients.begin()+i); - print_client(clients[i], "okay."); + print_client(clients[i], "done closing client from main process."); } } - else clients[i]->error=0; + else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } } } //TODO: at the end, server closes pipefd[1] for client @@ -229,7 +241,7 @@ int main(int argc, char* argv[]) void print_client(client_t* client, const char* what) { - fprintf(stderr,MSG_START " (client %s:%d) %s\n", inet_ntoa(client->addr.sin_addr), client->addr.sin_port, what); + fprintf(stderr,MSG_START "(client %s:%d) %s\n", inet_ntoa(client->addr.sin_addr), client->addr.sin_port, what); } void client_cleanup() @@ -245,7 +257,7 @@ void client() read(this_client->pipefd[0],buf,bufsizeall); if(send(this_client->socket,buf,bufsizeall,0)==-1) { - print_client(this_client, "closing."); + print_client(this_client, "client process is exiting."); exit(0); } } diff --git a/ddcd.h b/ddcd.h index be2a291..0767500 100644 --- a/ddcd.h +++ b/ddcd.h @@ -29,3 +29,4 @@ typedef struct client_s void client(); void error_exit(const char* why); void print_client(client_t* client, const char* what); +int proc_exists(pid_t pid); From edc2c21e436e1eb8f66a4e508204a43a45ca1481 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sat, 7 Nov 2015 19:21:08 +0100 Subject: [PATCH 08/74] FastDDC code added, and now it builds! --- csdr.c | 102 ++++++++++++++++++++++++++++++++++++- fastddc.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++ fastddc.h | 27 ++++++++++ libcsdr_wrapper.c | 1 + 4 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 fastddc.c create mode 100644 fastddc.h diff --git a/csdr.c b/csdr.c index 1f10e26..9445d6c 100644 --- a/csdr.c +++ b/csdr.c @@ -48,6 +48,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ima_adpcm.h" #include #include +#include +#include "fastddc.h" char usage[]= "csdr - a simple commandline tool for Software Defined Radio receiver DSP.\n\n" @@ -1269,7 +1271,6 @@ int main(int argc, char *argv[]) float high_cut; float transition_bw; window_t window = WINDOW_DEFAULT; - char window_string[256]; //TODO: nice buffer overflow opportunity int fd; if(fd=init_fifo(argc,argv)) @@ -1646,6 +1647,105 @@ int main(int argc, char *argv[]) } } + if( !strcmp(argv[1],"fastddc_fwd_cc") ) // [transition_bw [window]] + { + int decimation; + if(argc<=2) return badsyntax("need required parameter (decimation)"); + sscanf(argv[2],"%d",&decimation); + + float transition_bw = 0.05; + if(argc>=3) sscanf(argv[3],"%g",&transition_bw); + + window_t window = WINDOW_DEFAULT; + if(argc>=4) window=firdes_get_window_from_string(argv[5]); + else fprintf(stderr,"fastddc_fwd_cc: window = %s\n",firdes_get_string_from_window(window)); + + fastddc_t ddc; + if(fastddc_init(&ddc, transition_bw, decimation, 0)) { badsyntax("error in fastddc_init()"); return 1; } + fastddc_print(&ddc); + + if(!initialize_buffers()) return -2; + sendbufsize(ddc.fft_size); + + //make FFT plan + complexf* input = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_size); + complexf* windowed = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_size); + complexf* output = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_size); + + for(int i=0;i [transition_bw [window]] + { + int decimation; + if(argc<=2) return badsyntax("need required parameter (decimation)"); + sscanf(argv[2],"%d",&decimation); + + float shift_rate; + if(argc>=3) sscanf(argv[3],"%g",&shift_rate); + + float transition_bw = 0.05; + if(argc>=4) sscanf(argv[4],"%g",&transition_bw); + + window_t window = WINDOW_DEFAULT; + if(argc>=5) window=firdes_get_window_from_string(argv[5]); + else fprintf(stderr,"fastddc_apply_cc: window = %s\n",firdes_get_string_from_window(window)); + + fastddc_t ddc; + if(fastddc_init(&ddc, transition_bw, decimation, shift_rate)) { badsyntax("error in fastddc_init()"); return 1; } + fastddc_print(&ddc); + + if(!initialize_buffers()) return -2; + sendbufsize(ddc.output_size); + + //prepare making the filter and doing FFT on it + complexf* taps=(complexf*)calloc(sizeof(complexf),ddc.fft_size); //initialize to zero + complexf* taps_fft=(complexf*)malloc(sizeof(complexf)*ddc.fft_size); + FFT_PLAN_T* plan_taps = make_fft_c2c(ddc.fft_size, taps, taps_fft, 1, 0); //forward, don't benchmark (we need this only once) + + //make the filter + float filter_half_bw = 0.5/decimation; + firdes_bandpass_c(taps, ddc.taps_real_length, shift_rate-filter_half_bw, shift_rate+filter_half_bw, window); + fft_execute(plan_taps); + + //make FFT plan + complexf* input = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_size); + complexf* output = (complexf*)fft_malloc(sizeof(complexf)*ddc.output_size); + + int benchmark = 1; + if(benchmark) fprintf(stderr,"fastddc_apply_cc: benchmarking FFT..."); + FFT_PLAN_T* plan_inverse = make_fft_c2c(ddc.fft_size, input, output, 0, 1); //inverse, do benchmark + if(benchmark) fprintf(stderr," done\n"); + + decimating_shift_addition_status_t shift_stat; + bzero(&shift_stat, sizeof(shift_stat)); + for(;;) + { + FEOF_CHECK; + fread(input, sizeof(complexf), ddc.fft_size, stdin); + shift_stat = fastddc_apply_cc(input, output, &ddc, plan_inverse, taps_fft, shift_stat); + fwrite(output, sizeof(complexf), ddc.output_size, stdout); + TRY_YIELD; + } + } + if(!strcmp(argv[1],"none")) { return 0; diff --git a/fastddc.c b/fastddc.c new file mode 100644 index 0000000..40d5d21 --- /dev/null +++ b/fastddc.c @@ -0,0 +1,125 @@ +/* +This software is part of libcsdr, a set of simple DSP routines for +Software Defined Radio. + +Copyright (c) 2014, Andras Retzler +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL ANDRAS RETZLER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "fastddc.h" + +//DDC implementation based on: +//http://www.3db-labs.com/01598092_MultibandFilterbank.pdf + +inline int is_integer(float a) { return floorf(a) == a; } + +int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shift_rate) +{ + ddc->pre_decimation = 1; //this will be done in the frequency domain + ddc->post_decimation = decimation; //this will be done in the time domain + while( is_integer((float)ddc->post_decimation/2) && ddc->post_decimation/2 != 1) + { + ddc->post_decimation/=2; + ddc->pre_decimation*=2; + } + ddc->taps_real_length = firdes_filter_len(transition_bw); //the number of non-zero taps + ddc->taps_length = ceil(ddc->taps_real_length/(float)ddc->pre_decimation) * ddc->pre_decimation; //the number of taps must be a multiple of the decimation factor + ddc->fft_size = next_pow2(ddc->taps_length * 4); //it is a good rule of thumb for performance (based on the article), but we should do benchmarks + while (ddc->fft_sizepre_decimation) ddc->fft_size*=2; //fft_size should be a multiple of pre_decimation + ddc->overlap_length = ddc->taps_length - 1; + ddc->input_size = ddc->fft_size - ddc->overlap_length; + ddc->fft_inv_size = ddc->fft_size / ddc->pre_decimation; + + //Shift operation in the frequency domain: we can shift by a multiple of v. + ddc->v = ddc->fft_size/ddc->overlap_length; //+-1 ? (or maybe ceil() this?) //TODO: why? + int middlebin=ddc->fft_size / 2; + ddc->startbin = middlebin + middlebin * shift_rate * 2; + ddc->startbin = ddc->v * round( ddc->startbin / (float)ddc->v ); + ddc->offsetbin = ddc->startbin - middlebin; + ddc->post_shift = ((float)ddc->offsetbin/ddc->fft_size) - shift_rate; + ddc->pre_shift = ddc->offsetbin * ddc->v; + + //Overlap is scraped, not added + ddc->scrape=ddc->overlap_length/ddc->pre_decimation; + ddc->output_size=ddc->fft_inv_size-ddc->scrape; + + return ddc->fft_size<=2; //returns true on error +} + + +void fastddc_print(fastddc_t* ddc) +{ + fprintf(stderr, + "fastddc_print_sizes(): (fft_size = %d) = (taps_length = %d) + (input_size = %d) - 1\n" + "\t(overlap_length = %d) = taps_length - 1, taps_real_length = %d\n" + "\tdecimation = (pre_decimation = %d) * (post_decimation = %d), fft_inv_size = %d\n" + "\tstartbin = %d, offsetbin = %d, v = %d, pre_shift = %g, post_shift = %g\n" + , + ddc->fft_size, ddc->taps_length, ddc->input_size, + ddc->overlap_length, ddc->taps_real_length, + ddc->pre_decimation, ddc->post_decimation, ddc->fft_inv_size, + ddc->startbin, ddc->offsetbin, ddc->v, ddc->pre_shift, ddc->post_shift ); +} + +decimating_shift_addition_status_t fastddc_apply_cc(complexf* input, complexf* output, fastddc_t* ddc, FFT_PLAN_T* plan_inverse, complexf* taps_fft, decimating_shift_addition_status_t shift_stat) +{ + //implements DDC by using the overlap & scrape method + //TODO: +/-1s on overlap_size et al + //input shoud have ddc->fft_size number of elements + + complexf* inv_input = plan_inverse->input; + complexf* inv_output = plan_inverse->output; + + //Initialize buffers for inverse FFT to zero + for(int i=0;isize;i++) + { + iof(inv_input,i)=0; + qof(inv_input,i)=0; + } + + //Alias & shift & filter at once + // * no, we won't break this algorithm to parts that are easier to understand: now we go for speed + for(int i=0;ifft_size;i++) + { + int output_index = (ddc->startbin+i)%plan_inverse->size; + int tap_index = (ddc->fft_size+i-ddc->offsetbin)%ddc->fft_size; + cmultadd(inv_input+output_index, input+i, taps_fft+tap_index); //cmultadd(output, input1, input2): complex output += complex input1 * complex input 2 + } + + fft_execute(plan_inverse); + + //Normalize data + for(int i=0;isize;i++) //@apply_ddc_fft_cc: normalize by size + { + iof(inv_output,i)/=plan_inverse->size; + qof(inv_output,i)/=plan_inverse->size; + } + + //Overlap is scraped, not added + //Shift correction + shift_addition_data_t dsadata=decimating_shift_addition_init(ddc->post_shift, ddc->post_decimation); //this could be optimized (passed as parameter), but we would not win too much at all + shift_stat=decimating_shift_addition_cc(plan_inverse->output+ddc->scrape, output, ddc->output_size, dsadata, ddc->post_decimation, shift_stat); + return shift_stat; +} diff --git a/fastddc.h b/fastddc.h new file mode 100644 index 0000000..0029713 --- /dev/null +++ b/fastddc.h @@ -0,0 +1,27 @@ +#include +#include "libcsdr.h" +#include "libcsdr_gpl.h" + +typedef struct fastddc_s +{ + int pre_decimation; + int post_decimation; + int taps_length; + int taps_real_length; + int overlap_length; //it is taps_length - 1 + int fft_size; + int fft_inv_size; + int input_size; + int output_size; + float pre_shift; + int startbin; //for pre_shift + int v; //step for pre_shift + int offsetbin; + float post_shift; + int output_scrape; + int scrape; +} fastddc_t; + +int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shift_rate); +decimating_shift_addition_status_t fastddc_apply_cc(complexf* input, complexf* output, fastddc_t* ddc, FFT_PLAN_T* plan_inverse, complexf* taps_fft, decimating_shift_addition_status_t shift_stat); +void fastddc_print(fastddc_t* ddc); diff --git a/libcsdr_wrapper.c b/libcsdr_wrapper.c index 3e6ab87..ffa73f9 100644 --- a/libcsdr_wrapper.c +++ b/libcsdr_wrapper.c @@ -1,4 +1,5 @@ #include "libcsdr.c" #include "libcsdr_gpl.c" #include "ima_adpcm.c" +#include "fastddc.c" //this wrapper helps parsevect.py to generate better output From c7e363d5cf6fac6bc8ed1614b35407e35ae9b58d Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 17 Nov 2015 10:01:34 +0100 Subject: [PATCH 09/74] Fixed segmentation fault and a couple of other things. Added a GRC test for fastddc. --- Makefile | 4 +- csdr.c | 25 +- fastddc.c | 18 +- grc_tests/test_bandpass_fir_fft.grc | 1120 ++++++++++++++------------- grc_tests/test_fastddc.grc | 689 ++++++++++++++++ 5 files changed, 1295 insertions(+), 561 deletions(-) create mode 100644 grc_tests/test_fastddc.grc diff --git a/Makefile b/Makefile index 5de63bd..64f52ac 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ FFTW_PACKAGE = fftw-3.3.3 .PHONY: clean-vect clean all: csdr ddcd -libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c *.h +libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c fastddc.c *.h @echo NOTE: you may have to manually edit Makefile to optimize for your CPU \(especially if you compile on ARM, please edit PARAMS_NEON\). @echo Auto-detected optimization parameters: $(PARAMS_SIMD) @echo @@ -63,7 +63,7 @@ clean-vect: rm -f dumpvect*.vect clean: clean-vect rm -f libcsdr.so csdr ddcd -install: +install: all install -m 0755 libcsdr.so /usr/lib install -m 0755 csdr /usr/bin install -m 0755 csdr-fm /usr/bin diff --git a/csdr.c b/csdr.c index 9445d6c..55ce462 100644 --- a/csdr.c +++ b/csdr.c @@ -1649,15 +1649,16 @@ int main(int argc, char *argv[]) if( !strcmp(argv[1],"fastddc_fwd_cc") ) // [transition_bw [window]] { + int decimation; if(argc<=2) return badsyntax("need required parameter (decimation)"); sscanf(argv[2],"%d",&decimation); float transition_bw = 0.05; - if(argc>=3) sscanf(argv[3],"%g",&transition_bw); + if(argc>3) sscanf(argv[3],"%g",&transition_bw); window_t window = WINDOW_DEFAULT; - if(argc>=4) window=firdes_get_window_from_string(argv[5]); + if(argc>4) window=firdes_get_window_from_string(argv[5]); else fprintf(stderr,"fastddc_fwd_cc: window = %s\n",firdes_get_string_from_window(window)); fastddc_t ddc; @@ -1683,7 +1684,7 @@ int main(int argc, char *argv[]) { FEOF_CHECK; //overlapped FFT - for(int i=0;i=3) sscanf(argv[3],"%g",&shift_rate); + if(argc>3) sscanf(argv[3],"%g",&shift_rate); float transition_bw = 0.05; - if(argc>=4) sscanf(argv[4],"%g",&transition_bw); + if(argc>4) sscanf(argv[4],"%g",&transition_bw); window_t window = WINDOW_DEFAULT; - if(argc>=5) window=firdes_get_window_from_string(argv[5]); + if(argc>5) window=firdes_get_window_from_string(argv[5]); else fprintf(stderr,"fastddc_apply_cc: window = %s\n",firdes_get_string_from_window(window)); fastddc_t ddc; @@ -1726,14 +1727,16 @@ int main(int argc, char *argv[]) fft_execute(plan_taps); //make FFT plan + complexf* inv_input = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_inv_size); + complexf* inv_output = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_inv_size); + fprintf(stderr,"fastddc_apply_cc: benchmarking FFT..."); + FFT_PLAN_T* plan_inverse = make_fft_c2c(ddc.fft_inv_size, inv_input, inv_output, 0, 1); //inverse, do benchmark + fprintf(stderr," done\n"); + + //alloc. buffers complexf* input = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_size); complexf* output = (complexf*)fft_malloc(sizeof(complexf)*ddc.output_size); - int benchmark = 1; - if(benchmark) fprintf(stderr,"fastddc_apply_cc: benchmarking FFT..."); - FFT_PLAN_T* plan_inverse = make_fft_c2c(ddc.fft_size, input, output, 0, 1); //inverse, do benchmark - if(benchmark) fprintf(stderr," done\n"); - decimating_shift_addition_status_t shift_stat; bzero(&shift_stat, sizeof(shift_stat)); for(;;) diff --git a/fastddc.c b/fastddc.c index 40d5d21..baaf597 100644 --- a/fastddc.c +++ b/fastddc.c @@ -55,11 +55,11 @@ int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shif //Shift operation in the frequency domain: we can shift by a multiple of v. ddc->v = ddc->fft_size/ddc->overlap_length; //+-1 ? (or maybe ceil() this?) //TODO: why? int middlebin=ddc->fft_size / 2; - ddc->startbin = middlebin + middlebin * shift_rate * 2; + ddc->startbin = middlebin + middlebin * shift_rate * 2; ddc->startbin = ddc->v * round( ddc->startbin / (float)ddc->v ); ddc->offsetbin = ddc->startbin - middlebin; - ddc->post_shift = ((float)ddc->offsetbin/ddc->fft_size) - shift_rate; - ddc->pre_shift = ddc->offsetbin * ddc->v; + ddc->post_shift = shift_rate-((float)ddc->offsetbin/ddc->fft_size); + ddc->pre_shift = ddc->offsetbin/(float)ddc->fft_size; //Overlap is scraped, not added ddc->scrape=ddc->overlap_length/ddc->pre_decimation; @@ -73,14 +73,16 @@ void fastddc_print(fastddc_t* ddc) { fprintf(stderr, "fastddc_print_sizes(): (fft_size = %d) = (taps_length = %d) + (input_size = %d) - 1\n" - "\t(overlap_length = %d) = taps_length - 1, taps_real_length = %d\n" - "\tdecimation = (pre_decimation = %d) * (post_decimation = %d), fft_inv_size = %d\n" - "\tstartbin = %d, offsetbin = %d, v = %d, pre_shift = %g, post_shift = %g\n" + " overlap :: (overlap_length = %d) = taps_length - 1, taps_real_length = %d\n" + " decimation :: decimation = (pre_decimation = %d) * (post_decimation = %d), fft_inv_size = %d\n" + " shift :: startbin = %d, offsetbin = %d, v = %d, pre_shift = %g, post_shift = %g\n" + " o&s :: output_size = %d, scrape = %d\n" , ddc->fft_size, ddc->taps_length, ddc->input_size, ddc->overlap_length, ddc->taps_real_length, ddc->pre_decimation, ddc->post_decimation, ddc->fft_inv_size, - ddc->startbin, ddc->offsetbin, ddc->v, ddc->pre_shift, ddc->post_shift ); + ddc->startbin, ddc->offsetbin, ddc->v, ddc->pre_shift, ddc->post_shift, + ddc->output_size, ddc->scrape ); } decimating_shift_addition_status_t fastddc_apply_cc(complexf* input, complexf* output, fastddc_t* ddc, FFT_PLAN_T* plan_inverse, complexf* taps_fft, decimating_shift_addition_status_t shift_stat) @@ -120,6 +122,6 @@ decimating_shift_addition_status_t fastddc_apply_cc(complexf* input, complexf* o //Overlap is scraped, not added //Shift correction shift_addition_data_t dsadata=decimating_shift_addition_init(ddc->post_shift, ddc->post_decimation); //this could be optimized (passed as parameter), but we would not win too much at all - shift_stat=decimating_shift_addition_cc(plan_inverse->output+ddc->scrape, output, ddc->output_size, dsadata, ddc->post_decimation, shift_stat); + shift_stat=decimating_shift_addition_cc(inv_output+ddc->scrape, output, ddc->output_size, dsadata, ddc->post_decimation, shift_stat); return shift_stat; } diff --git a/grc_tests/test_bandpass_fir_fft.grc b/grc_tests/test_bandpass_fir_fft.grc index 0d2bc4d..8f010b3 100644 --- a/grc_tests/test_bandpass_fir_fft.grc +++ b/grc_tests/test_bandpass_fir_fft.grc @@ -1,49 +1,49 @@ - - + + Sat Nov 22 11:19:26 2014 options - - id - top_block - - - _enabled - True - - - title - - author - - description - - window_size 1280, 1024 - - generate_options - wx_gui - category Custom - run_options - prompt + comment + - run + description + + + + _enabled True + + _coordinate + (24, 11) + + + _rotation + 0 + + + generate_options + wx_gui + + + id + top_block + max_nouts 0 @@ -53,36 +53,91 @@ - alias + run_options + prompt + + + run + True + + + thread_safe_setters - _coordinate - (24, 11) - - - _rotation - 0 + title + - variable + variable_slider - id - samp_rate + comment + + + + converver + float_converter + + + value + 1e3 _enabled True - value - 24e3 + _coordinate + (8, 83) - alias + _rotation + 0 + + + grid_pos + + id + freq + + + label + + + + max + samp_rate/2 + + + min + -samp_rate/2 + + + notebook + + + + num_steps + 100 + + + style + wx.SL_HORIZONTAL + + + + variable + + comment + + + + _enabled + True + _coordinate (184, 11) @@ -91,16 +146,64 @@ _rotation 0 + + id + samp_rate + + + value + 24e3 + analog_sig_source_x + + amp + 1 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + freq + freq + + + _coordinate + (128, 83) + + + _rotation + 0 + id analog_sig_source_x_0 - _enabled - True + maxoutbuf + 0 + + + minoutbuf + 0 + + + offset + 0 type @@ -114,84 +217,24 @@ waveform analog.GR_COS_WAVE - - freq - freq - - - amp - 1 - - - offset - 0 - - - alias - - - - affinity - - - - minoutbuf - 0 - - - maxoutbuf - 0 - - - _coordinate - (128, 83) - - - _rotation - 0 - blocks_throttle - - id - blocks_throttle_0 - - - _enabled - True - - - type - complex - - - samples_per_second - samp_rate - - - vlen - 1 - - - ignoretag - True - alias + + comment + + affinity - minoutbuf - 0 - - - maxoutbuf - 0 + _enabled + True _coordinate @@ -201,460 +244,96 @@ _rotation 0 - - - variable_slider id - freq + blocks_throttle_0 - _enabled + ignoretag True - label - - - - value - 1e3 - - - min - -samp_rate/2 - - - max - samp_rate/2 - - - num_steps - 100 - - - style - wx.SL_HORIZONTAL - - - converver - float_converter - - - grid_pos - - - - notebook - - - - alias - - - - _coordinate - (8, 83) - - - _rotation + maxoutbuf 0 - - - wxgui_fftsink2 - id - wxgui_fftsink2_0_0 + minoutbuf + 0 - _enabled - True + samples_per_second + samp_rate type complex - title - FFT Plot of original signal - - - samp_rate - samp_rate - - - baseband_freq - 0 - - - y_per_div - 10 - - - y_divs - 10 - - - ref_level - 0 - - - ref_scale - 2.0 - - - fft_size - 1024 - - - fft_rate - 15 - - - peak_hold - False - - - average - False - - - avg_alpha - 0 - - - win - None - - - win_size - - - - grid_pos - - - - notebook - nb0,1 - - - freqvar - None - - - alias - - - - affinity - - - - _coordinate - (504, 219) - - - _rotation - 0 - - - - wxgui_scopesink2 - - id - wxgui_scopesink2_0 - - - _enabled - True - - - type - complex - - - title - Scope Plot of original signal - - - samp_rate - samp_rate - - - v_scale - 0 - - - v_offset - 0 - - - t_scale - 0 - - - ac_couple - False - - - xy_mode - False - - - num_inputs + vlen 1 - - win_size - - - - grid_pos - - - - notebook - nb0,0 - - - trig_mode - wxgui.TRIG_MODE_AUTO - - - y_axis_label - Counts - + + + ha5kfu_execproc_xx alias - affinity + commandline + csdr bandpass_fir_fft_cc 0.1 0.5 0.05 HAMMING + + + comment - _coordinate - (504, 435) - - - _rotation - 0 - - - - wxgui_scopesink2 - - id - wxgui_scopesink2_0_0 + affinity + _enabled True - - type - complex - - - title - Scope Plot of new signal - - - samp_rate - samp_rate - - - v_scale - 0 - - - v_offset - 0 - - - t_scale - 0 - - - ac_couple - False - - - xy_mode - False - - - num_inputs - 1 - - - win_size - - - - grid_pos - - - - notebook - nb1,0 - - - trig_mode - wxgui.TRIG_MODE_AUTO - - - y_axis_label - Counts - - - alias - - - - affinity - - _coordinate - (784, 283) + (488, 147) _rotation 0 - - - wxgui_fftsink2 id - wxgui_fftsink2_0 + ha5kfu_execproc_xx_0 - _enabled - True + maxoutbuf + 0 + + + minoutbuf + 0 type - complex - - - title - FFT Plot of new signal - - - samp_rate - samp_rate - - - baseband_freq - 0 - - - y_per_div - 10 - - - y_divs - 10 - - - ref_level - 0 - - - ref_scale - 2.0 - - - fft_size - 1024 - - - fft_rate - 15 - - - peak_hold - False - - - average - False - - - avg_alpha - 0 - - - win - None - - - win_size - - - - grid_pos - - - - notebook - nb1,1 - - - freqvar - None - - - alias - - - - affinity - - - - _coordinate - (784, 67) - - - _rotation - 0 + cc notebook - id - nb0 + alias + + + + comment + _enabled True - - style - wx.NB_TOP - - - labels - ['scope', 'fft'] - - - grid_pos - 1,1,1,1 - - - notebook - - - - alias - - _coordinate (272, 11) @@ -663,37 +342,41 @@ _rotation 0 - - - notebook + + grid_pos + 1,1,1,1 + id - nb1 - - - _enabled - True - - - style - wx.NB_TOP + nb0 labels ['scope', 'fft'] - - grid_pos - 1,2,1,1 - notebook + + style + wx.NB_TOP + + + + notebook alias + + comment + + + + _enabled + True + _coordinate (416, 11) @@ -702,49 +385,406 @@ _rotation 0 - - - ha5kfu_execproc_xx + + grid_pos + 1,2,1,1 + id - ha5kfu_execproc_xx_0 + nb1 - _enabled - True + labels + ['scope', 'fft'] - type - cc + notebook + - commandline - csdr bandpass_fir_fft_cc 0.1 0.5 0.05 HAMMING + style + wx.NB_TOP + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 alias + + comment + + affinity - minoutbuf - 0 + _enabled + True - maxoutbuf - 0 + fft_size + 1024 + + + freqvar + None _coordinate - (488, 147) + (784, 67) _rotation 0 + + grid_pos + + + + id + wxgui_fftsink2_0 + + + notebook + nb1,1 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT Plot of new signal + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (504, 219) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0_0 + + + notebook + nb0,1 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT Plot of original signal + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (504, 435) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_scopesink2_0 + + + notebook + nb0,0 + + + num_inputs + 1 + + + samp_rate + samp_rate + + + t_scale + 0 + + + title + Scope Plot of original signal + + + trig_mode + wxgui.TRIG_MODE_AUTO + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + + + + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (784, 283) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_scopesink2_0_0 + + + notebook + nb1,0 + + + num_inputs + 1 + + + samp_rate + samp_rate + + + t_scale + 0 + + + title + Scope Plot of new signal + + + trig_mode + wxgui.TRIG_MODE_AUTO + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + analog_sig_source_x_0 @@ -758,12 +798,6 @@ 0 0 - - ha5kfu_execproc_xx_0 - wxgui_fftsink2_0 - 0 - 0 - blocks_throttle_0 wxgui_fftsink2_0_0 @@ -776,6 +810,12 @@ 0 0 + + ha5kfu_execproc_xx_0 + wxgui_fftsink2_0 + 0 + 0 + ha5kfu_execproc_xx_0 wxgui_scopesink2_0_0 diff --git a/grc_tests/test_fastddc.grc b/grc_tests/test_fastddc.grc new file mode 100644 index 0000000..5c3706e --- /dev/null +++ b/grc_tests/test_fastddc.grc @@ -0,0 +1,689 @@ + + + + Sat Nov 15 20:06:19 2014 + + options + + author + + + + window_size + 1280, 1024 + + + category + Custom + + + comment + + + + description + + + + _enabled + True + + + _coordinate + (10, 10) + + + _rotation + 0 + + + generate_options + wx_gui + + + id + top_block + + + max_nouts + 0 + + + realtime_scheduling + + + + run_options + prompt + + + run + True + + + thread_safe_setters + + + + title + + + + + variable_slider + + comment + + + + converver + float_converter + + + value + 50 + + + _enabled + True + + + _coordinate + (16, 267) + + + _rotation + 0 + + + grid_pos + + + + id + freq + + + label + + + + max + samp_rate/2 + + + min + -samp_rate/2 + + + notebook + + + + num_steps + 100 + + + style + wx.SL_HORIZONTAL + + + + variable + + comment + + + + _enabled + True + + + _coordinate + (9, 170) + + + _rotation + 0 + + + id + samp_rate + + + value + 250000 + + + + analog_sig_source_x + + amp + 0.2 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + freq + freq + + + _coordinate + (224, 29) + + + _rotation + 0 + + + id + analog_sig_source_x_0 + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + offset + 0 + + + type + complex + + + samp_rate + samp_rate + + + waveform + analog.GR_COS_WAVE + + + + blocks_throttle + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (424, 59) + + + _rotation + 0 + + + id + blocks_throttle_0 + + + ignoretag + True + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + samples_per_second + samp_rate + + + type + complex + + + vlen + 1 + + + + ha5kfu_execproc_xx + + alias + + + + commandline + csdr fastddc_fwd_cc 10 | csdr fastddc_apply_cc 10 0.1 | csdr floatdump_f 2>&1 | head -n 100 + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (584, 59) + + + _rotation + 0 + + + id + ha5kfu_execproc_xx_1 + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + type + cc + + + + notebook + + alias + + + + comment + + + + _enabled + True + + + _coordinate + (160, 283) + + + _rotation + 0 + + + grid_pos + + + + id + nb + + + labels + ['FFT', 'Scope'] + + + notebook + + + + style + wx.NB_TOP + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (936, 155) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0 + + + notebook + nb, 0 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT plot of csdr processed signal + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (936, 371) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0_0 + + + notebook + + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT plot of original signal + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (952, 35) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_scopesink2_0 + + + notebook + nb, 1 + + + num_inputs + 1 + + + samp_rate + samp_rate + + + t_scale + 0 + + + title + Scope plot of csdr processed signal + + + trig_mode + wxgui.TRIG_MODE_AUTO + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + + + + analog_sig_source_x_0 + blocks_throttle_0 + 0 + 0 + + + blocks_throttle_0 + ha5kfu_execproc_xx_1 + 0 + 0 + + + blocks_throttle_0 + wxgui_fftsink2_0_0 + 0 + 0 + + + ha5kfu_execproc_xx_1 + wxgui_fftsink2_0 + 0 + 0 + + + ha5kfu_execproc_xx_1 + wxgui_scopesink2_0 + 0 + 0 + + From 175c700cc9ce2e84922eab50f6019ab27ce9c82a Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 17 Nov 2015 21:54:06 +0100 Subject: [PATCH 10/74] Verified that fastddc_fwd_cc works. --- csdr.c | 35 +++++++++++++++++++++++++++++++---- fastddc.c | 22 +++++++++++++++++++--- fastddc.h | 2 +- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/csdr.c b/csdr.c index 55ce462..fc46ee4 100644 --- a/csdr.c +++ b/csdr.c @@ -697,7 +697,7 @@ int main(int argc, char *argv[]) { FEOF_CHECK; FREAD_R; - for(int i=0; i3) sscanf(argv[3],"%g",&transition_bw); window_t window = WINDOW_DEFAULT; - if(argc>4) window=firdes_get_window_from_string(argv[5]); + if(argc>4) window=firdes_get_window_from_string(argv[4]); else fprintf(stderr,"fastddc_fwd_cc: window = %s\n",firdes_get_string_from_window(window)); fastddc_t ddc; @@ -1693,7 +1693,7 @@ int main(int argc, char *argv[]) } } - if( !strcmp(argv[1],"fastddc_apply_cc") ) // [transition_bw [window]] + if( !strcmp(argv[1],"fastddc_inv_cc") ) // [transition_bw [window]] { int decimation; if(argc<=2) return badsyntax("need required parameter (decimation)"); @@ -1743,12 +1743,39 @@ int main(int argc, char *argv[]) { FEOF_CHECK; fread(input, sizeof(complexf), ddc.fft_size, stdin); - shift_stat = fastddc_apply_cc(input, output, &ddc, plan_inverse, taps_fft, shift_stat); + shift_stat = fastddc_inv_cc(input, output, &ddc, plan_inverse, taps_fft, shift_stat); fwrite(output, sizeof(complexf), ddc.output_size, stdout); TRY_YIELD; } } + if( !strcmp(argv[1], "_fft2octave") ) + { + int fft_size; + if(argc<=2) return badsyntax("need required parameter (fft_size)"); + sscanf(argv[2],"%d",&fft_size); + + complexf* fft_input=(complexf*)malloc(sizeof(complexf)*fft_size); + initialize_buffers(); + if(!sendbufsize(fft_size)) return -2; + + printf("setenv(\"GNUTERM\",\"X11 noraise\");y=zeros(1,%d);semilogy(y,\"ydatasource\",\"y\");\n",fft_size); + for(;;) + { + FEOF_CHECK; + fread(fft_input, sizeof(complexf), fft_size, stdin); + printf("fftdata=["); + //we have to swap the two parts of the array to get a valid spectrum + for(int i=fft_size/2;ioutput_size, ddc->scrape ); } -decimating_shift_addition_status_t fastddc_apply_cc(complexf* input, complexf* output, fastddc_t* ddc, FFT_PLAN_T* plan_inverse, complexf* taps_fft, decimating_shift_addition_status_t shift_stat) +void fft_swap_sides(complexf* io, int fft_size) +{ + int middle=fft_size/2; + complexf temp; + for(int i=0;ifft_size); //this is not very optimal, but now we stick with this slow solution until we got the algorithm working for(int i=0;ifft_size;i++) { int output_index = (ddc->startbin+i)%plan_inverse->size; @@ -111,9 +126,10 @@ decimating_shift_addition_status_t fastddc_apply_cc(complexf* input, complexf* o } fft_execute(plan_inverse); + fft_swap_sides(inv_output,plan_inverse->size); //Normalize data - for(int i=0;isize;i++) //@apply_ddc_fft_cc: normalize by size + for(int i=0;isize;i++) //@fastddc_inv_cc: normalize by size { iof(inv_output,i)/=plan_inverse->size; qof(inv_output,i)/=plan_inverse->size; diff --git a/fastddc.h b/fastddc.h index 0029713..e4aae2e 100644 --- a/fastddc.h +++ b/fastddc.h @@ -23,5 +23,5 @@ typedef struct fastddc_s } fastddc_t; int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shift_rate); -decimating_shift_addition_status_t fastddc_apply_cc(complexf* input, complexf* output, fastddc_t* ddc, FFT_PLAN_T* plan_inverse, complexf* taps_fft, decimating_shift_addition_status_t shift_stat); +decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* output, fastddc_t* ddc, FFT_PLAN_T* plan_inverse, complexf* taps_fft, decimating_shift_addition_status_t shift_stat); void fastddc_print(fastddc_t* ddc); From a6cf61d60be5144d2f9c70c96af81194aa14d7d7 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 18 Nov 2015 17:47:18 +0100 Subject: [PATCH 11/74] Fixed output_size (there were a bunch of zeros in the output). --- csdr.c | 6 +++--- fastddc.c | 10 ++++++---- fastddc.h | 2 +- grc_tests/test_fastddc.grc | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/csdr.c b/csdr.c index fc46ee4..0e02ceb 100644 --- a/csdr.c +++ b/csdr.c @@ -1714,7 +1714,7 @@ int main(int argc, char *argv[]) fastddc_print(&ddc); if(!initialize_buffers()) return -2; - sendbufsize(ddc.output_size); + sendbufsize(ddc.post_input_size/ddc.post_decimation); //TODO not exactly correct //prepare making the filter and doing FFT on it complexf* taps=(complexf*)calloc(sizeof(complexf),ddc.fft_size); //initialize to zero @@ -1735,7 +1735,7 @@ int main(int argc, char *argv[]) //alloc. buffers complexf* input = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_size); - complexf* output = (complexf*)fft_malloc(sizeof(complexf)*ddc.output_size); + complexf* output = (complexf*)fft_malloc(sizeof(complexf)*ddc.post_input_size); decimating_shift_addition_status_t shift_stat; bzero(&shift_stat, sizeof(shift_stat)); @@ -1744,7 +1744,7 @@ int main(int argc, char *argv[]) FEOF_CHECK; fread(input, sizeof(complexf), ddc.fft_size, stdin); shift_stat = fastddc_inv_cc(input, output, &ddc, plan_inverse, taps_fft, shift_stat); - fwrite(output, sizeof(complexf), ddc.output_size, stdout); + fwrite(output, sizeof(complexf), shift_stat.output_size, stdout); TRY_YIELD; } } diff --git a/fastddc.c b/fastddc.c index 7a76234..4cd45ae 100644 --- a/fastddc.c +++ b/fastddc.c @@ -63,7 +63,7 @@ int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shif //Overlap is scraped, not added ddc->scrape=ddc->overlap_length/ddc->pre_decimation; - ddc->output_size=ddc->fft_inv_size-ddc->scrape; + ddc->post_input_size=ddc->fft_inv_size-ddc->scrape; return ddc->fft_size<=2; //returns true on error } @@ -76,13 +76,13 @@ void fastddc_print(fastddc_t* ddc) " overlap :: (overlap_length = %d) = taps_length - 1, taps_real_length = %d\n" " decimation :: decimation = (pre_decimation = %d) * (post_decimation = %d), fft_inv_size = %d\n" " shift :: startbin = %d, offsetbin = %d, v = %d, pre_shift = %g, post_shift = %g\n" - " o&s :: output_size = %d, scrape = %d\n" + " o&s :: post_input_size = %d, scrape = %d\n" , ddc->fft_size, ddc->taps_length, ddc->input_size, ddc->overlap_length, ddc->taps_real_length, ddc->pre_decimation, ddc->post_decimation, ddc->fft_inv_size, ddc->startbin, ddc->offsetbin, ddc->v, ddc->pre_shift, ddc->post_shift, - ddc->output_size, ddc->scrape ); + ddc->post_input_size, ddc->scrape ); } void fft_swap_sides(complexf* io, int fft_size) @@ -138,6 +138,8 @@ decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* out //Overlap is scraped, not added //Shift correction shift_addition_data_t dsadata=decimating_shift_addition_init(ddc->post_shift, ddc->post_decimation); //this could be optimized (passed as parameter), but we would not win too much at all - shift_stat=decimating_shift_addition_cc(inv_output+ddc->scrape, output, ddc->output_size, dsadata, ddc->post_decimation, shift_stat); + shift_stat=decimating_shift_addition_cc(inv_output+ddc->scrape, output, ddc->post_input_size, dsadata, ddc->post_decimation, shift_stat); + + //memcpy(inv_output+ddc->scrape, return shift_stat; } diff --git a/fastddc.h b/fastddc.h index e4aae2e..1e27cef 100644 --- a/fastddc.h +++ b/fastddc.h @@ -12,7 +12,7 @@ typedef struct fastddc_s int fft_size; int fft_inv_size; int input_size; - int output_size; + int post_input_size; float pre_shift; int startbin; //for pre_shift int v; //step for pre_shift diff --git a/grc_tests/test_fastddc.grc b/grc_tests/test_fastddc.grc index 5c3706e..79d064f 100644 --- a/grc_tests/test_fastddc.grc +++ b/grc_tests/test_fastddc.grc @@ -281,7 +281,7 @@ commandline - csdr fastddc_fwd_cc 10 | csdr fastddc_apply_cc 10 0.1 | csdr floatdump_f 2>&1 | head -n 100 + csdr fastddc_fwd_cc 10 | csdr fastddc_inv_cc 10 0.1 comment From 895bc2040982831cd38e2df4e29ae448f4efcbde Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Fri, 20 Nov 2015 08:21:09 +0100 Subject: [PATCH 12/74] Fixed fastddc_fwd_cc :-) --- csdr.c | 3 ++- grc_tests/test_fastddc.grc | 43 +++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/csdr.c b/csdr.c index 0e02ceb..4d2fc25 100644 --- a/csdr.c +++ b/csdr.c @@ -1684,9 +1684,10 @@ int main(int argc, char *argv[]) { FEOF_CHECK; //overlapped FFT - for(int i=0;i + + variable + + comment + + + + _enabled + True + + + _coordinate + (128, 179) + + + _rotation + 0 + + + id + decimation + + + value + 4 + + variable_slider @@ -238,7 +265,7 @@ _coordinate - (424, 59) + (424, 235) _rotation @@ -281,7 +308,7 @@ commandline - csdr fastddc_fwd_cc 10 | csdr fastddc_inv_cc 10 0.1 + csdr fastddc_fwd_cc 4 | csdr fastddc_inv_cc 4 0.1 comment @@ -297,7 +324,7 @@ _coordinate - (584, 59) + (616, 235) _rotation @@ -403,7 +430,7 @@ _coordinate - (936, 155) + (952, 155) _rotation @@ -439,7 +466,7 @@ samp_rate - samp_rate + samp_rate/decimation title @@ -494,7 +521,7 @@ _enabled - True + 1 fft_size @@ -506,7 +533,7 @@ _coordinate - (936, 371) + (616, 379) _rotation @@ -617,7 +644,7 @@ samp_rate - samp_rate + samp_rate/decimation t_scale From 646de8b258446e9fccb2c3fbea0654d5096d709a Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sat, 21 Nov 2015 19:52:35 +0100 Subject: [PATCH 13/74] scrape -> scrap, fixed shift_math_cc, found a bug at fastddc.c line 68 which still waits to be fixed --- csdr.c | 11 ++--- fastddc.c | 31 +++++++------- fastddc.h | 5 ++- grc_tests/test_bandpass_fir_fft.grc | 65 ++++++++++++++++++++++++++++- 4 files changed, 88 insertions(+), 24 deletions(-) diff --git a/csdr.c b/csdr.c index 4d2fc25..0936d56 100644 --- a/csdr.c +++ b/csdr.c @@ -452,6 +452,7 @@ int main(int argc, char *argv[]) { FEOF_CHECK; if(!FREAD_C) break; + starting_phase=shift_math_cc((complexf*)input_buffer, (complexf*)output_buffer, the_bufsize, rate, starting_phase); FWRITE_C; TRY_YIELD; } @@ -1295,7 +1296,7 @@ int main(int argc, char *argv[]) if (fft_size-taps_length<200) fft_size<<=1; int input_size = fft_size - taps_length + 1; int overlap_length = taps_length - 1; - fprintf(stderr,"bandpass_fir_fft_cc: (fft_size = %d) = (taps_length = %d) + (input_size = %d) - 1\n(overlap_length = %d) = taps_length - 1\n", fft_size, taps_length, input_size, overlap_length); + fprintf(stderr,"bandpass_fir_fft_cc: (fft_size = %d) = (taps_length = %d) + (input_size = %d) - 1\n(overlap_length = %d) = taps_length - 1\n", fft_size, taps_length, input_size, overlap_length ); if (fft_size<=2) return badsyntax("FFT size error."); if(!sendbufsize(getbufsize())) return -2; @@ -1663,7 +1664,7 @@ int main(int argc, char *argv[]) fastddc_t ddc; if(fastddc_init(&ddc, transition_bw, decimation, 0)) { badsyntax("error in fastddc_init()"); return 1; } - fastddc_print(&ddc); + fastddc_print(&ddc,"fastddc_fwd_cc"); if(!initialize_buffers()) return -2; sendbufsize(ddc.fft_size); @@ -1686,8 +1687,8 @@ int main(int argc, char *argv[]) //overlapped FFT for(int i=0;iv = ddc->fft_size/ddc->overlap_length; //+-1 ? (or maybe ceil() this?) //TODO: why? int middlebin=ddc->fft_size / 2; - ddc->startbin = middlebin + middlebin * shift_rate * 2; + ddc->startbin = middlebin + middlebin * shift_rate * 2; + //fprintf(stderr, "ddc->startbin=%g\n",(float)ddc->startbin); ddc->startbin = ddc->v * round( ddc->startbin / (float)ddc->v ); + //fprintf(stderr, "ddc->startbin=%g\n",(float)ddc->startbin); ddc->offsetbin = ddc->startbin - middlebin; ddc->post_shift = shift_rate-((float)ddc->offsetbin/ddc->fft_size); ddc->pre_shift = ddc->offsetbin/(float)ddc->fft_size; + ddc->dsadata = decimating_shift_addition_init(ddc->post_shift, ddc->post_decimation); - //Overlap is scraped, not added - ddc->scrape=ddc->overlap_length/ddc->pre_decimation; - ddc->post_input_size=ddc->fft_inv_size-ddc->scrape; + //Overlap is scrapd, not added + ddc->scrap=ddc->overlap_length/ddc->pre_decimation; //TODO this is problematic sometimes! overlap_length = 401 :: scrap = 200 + ddc->post_input_size=ddc->fft_inv_size-ddc->scrap; return ddc->fft_size<=2; //returns true on error } -void fastddc_print(fastddc_t* ddc) +void fastddc_print(fastddc_t* ddc, char* source) { fprintf(stderr, - "fastddc_print_sizes(): (fft_size = %d) = (taps_length = %d) + (input_size = %d) - 1\n" + "%s: fastddc_print_sizes(): (fft_size = %d) = (taps_length = %d) + (input_size = %d) - 1\n" " overlap :: (overlap_length = %d) = taps_length - 1, taps_real_length = %d\n" " decimation :: decimation = (pre_decimation = %d) * (post_decimation = %d), fft_inv_size = %d\n" " shift :: startbin = %d, offsetbin = %d, v = %d, pre_shift = %g, post_shift = %g\n" - " o&s :: post_input_size = %d, scrape = %d\n" + " o&s :: post_input_size = %d, scrap = %d\n" , - ddc->fft_size, ddc->taps_length, ddc->input_size, + source, ddc->fft_size, ddc->taps_length, ddc->input_size, ddc->overlap_length, ddc->taps_real_length, ddc->pre_decimation, ddc->post_decimation, ddc->fft_inv_size, ddc->startbin, ddc->offsetbin, ddc->v, ddc->pre_shift, ddc->post_shift, - ddc->post_input_size, ddc->scrape ); + ddc->post_input_size, ddc->scrap ); } void fft_swap_sides(complexf* io, int fft_size) @@ -102,7 +105,7 @@ void fft_swap_sides(complexf* io, int fft_size) decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* output, fastddc_t* ddc, FFT_PLAN_T* plan_inverse, complexf* taps_fft, decimating_shift_addition_status_t shift_stat) { - //implements DDC by using the overlap & scrape method + //implements DDC by using the overlap & scrap method //TODO: +/-1s on overlap_size et al //input shoud have ddc->fft_size number of elements @@ -135,11 +138,9 @@ decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* out qof(inv_output,i)/=plan_inverse->size; } - //Overlap is scraped, not added + //Overlap is scrapped, not added //Shift correction - shift_addition_data_t dsadata=decimating_shift_addition_init(ddc->post_shift, ddc->post_decimation); //this could be optimized (passed as parameter), but we would not win too much at all - shift_stat=decimating_shift_addition_cc(inv_output+ddc->scrape, output, ddc->post_input_size, dsadata, ddc->post_decimation, shift_stat); - - //memcpy(inv_output+ddc->scrape, + shift_stat=decimating_shift_addition_cc(inv_output+ddc->scrap, output, ddc->post_input_size, ddc->dsadata, ddc->post_decimation, shift_stat); + //memcpy(inv_output+ddc->scrap, output return shift_stat; } diff --git a/fastddc.h b/fastddc.h index 1e27cef..d6e15d1 100644 --- a/fastddc.h +++ b/fastddc.h @@ -19,9 +19,10 @@ typedef struct fastddc_s int offsetbin; float post_shift; int output_scrape; - int scrape; + int scrap; + shift_addition_data_t dsadata; } fastddc_t; int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shift_rate); decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* output, fastddc_t* ddc, FFT_PLAN_T* plan_inverse, complexf* taps_fft, decimating_shift_addition_status_t shift_stat); -void fastddc_print(fastddc_t* ddc); +void fastddc_print(fastddc_t* ddc, char* source); diff --git a/grc_tests/test_bandpass_fir_fft.grc b/grc_tests/test_bandpass_fir_fft.grc index 8f010b3..a1cc5f4 100644 --- a/grc_tests/test_bandpass_fir_fft.grc +++ b/grc_tests/test_bandpass_fir_fft.grc @@ -155,6 +155,61 @@ 24e3 + + analog_noise_source_x + + amp + 1 + + + alias + + + + comment + + + + affinity + + + + _enabled + 0 + + + _coordinate + (112, 219) + + + _rotation + 0 + + + id + analog_noise_source_x_0 + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + noise_type + analog.GR_GAUSSIAN + + + type + complex + + + seed + 0 + + analog_sig_source_x @@ -175,7 +230,7 @@ _enabled - True + 1 freq @@ -281,7 +336,7 @@ commandline - csdr bandpass_fir_fft_cc 0.1 0.5 0.05 HAMMING + csdr bandpass_fir_fft_cc 0.1 0.3 0.05 HAMMING comment @@ -786,6 +841,12 @@ Counts + + analog_noise_source_x_0 + blocks_throttle_0 + 0 + 0 + analog_sig_source_x_0 blocks_throttle_0 From b6fde4ce8632e71dad32fbd95c6044fe52366090 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sun, 22 Nov 2015 14:34:24 +0100 Subject: [PATCH 14/74] Now I've got the Overlap & Scrap right! The spectrum is clean. --- csdr.c | 6 ++++-- fastddc.c | 48 ++++++++++++++++++++++++++++++++++-------------- fastddc.h | 2 +- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/csdr.c b/csdr.c index 0936d56..c7ada5b 100644 --- a/csdr.c +++ b/csdr.c @@ -1725,13 +1725,14 @@ int main(int argc, char *argv[]) //make the filter float filter_half_bw = 0.5/decimation; - firdes_bandpass_c(taps, ddc.taps_real_length, shift_rate-filter_half_bw, shift_rate+filter_half_bw, window); + fprintf(stderr, "fastddc_inv_cc: preparing a bandpass filter of [%g, %g] cutoff rates. Real transition bandwidth is: %g\n", shift_rate-filter_half_bw, shift_rate+filter_half_bw, 4.0/ddc.taps_length); + firdes_bandpass_c(taps, ddc.taps_length, shift_rate-filter_half_bw, shift_rate+filter_half_bw, window); fft_execute(plan_taps); //make FFT plan complexf* inv_input = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_inv_size); complexf* inv_output = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_inv_size); - fprintf(stderr,"fastddc_apply_cc: benchmarking FFT..."); + fprintf(stderr,"fastddc_inv_cc: benchmarking FFT..."); FFT_PLAN_T* plan_inverse = make_fft_c2c(ddc.fft_inv_size, inv_input, inv_output, 0, 1); //inverse, do benchmark fprintf(stderr," done\n"); @@ -1747,6 +1748,7 @@ int main(int argc, char *argv[]) fread(input, sizeof(complexf), ddc.fft_size, stdin); shift_stat = fastddc_inv_cc(input, output, &ddc, plan_inverse, taps_fft, shift_stat); fwrite(output, sizeof(complexf), shift_stat.output_size, stdout); + fprintf(stderr, "ss os = %d\n", shift_stat.output_size); TRY_YIELD; } } diff --git a/fastddc.c b/fastddc.c index 523d800..9aab721 100644 --- a/fastddc.c +++ b/fastddc.c @@ -44,16 +44,16 @@ int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shif ddc->post_decimation/=2; ddc->pre_decimation*=2; } - ddc->taps_real_length = firdes_filter_len(transition_bw); //the number of non-zero taps - ddc->taps_length = ceil(ddc->taps_real_length/(float)ddc->pre_decimation) * ddc->pre_decimation; //the number of taps must be a multiple of the decimation factor + ddc->taps_min_length = firdes_filter_len(transition_bw); //his is the minimal number of taps to achieve the given transition_bw; we are likely to have more taps than this number. + ddc->taps_length = next_pow2(ceil(ddc->taps_min_length/(float)ddc->pre_decimation) * ddc->pre_decimation) + 1; //the number of taps must be a multiple of the decimation factor ddc->fft_size = next_pow2(ddc->taps_length * 4); //it is a good rule of thumb for performance (based on the article), but we should do benchmarks - while (ddc->fft_sizepre_decimation) ddc->fft_size*=2; //fft_size should be a multiple of pre_decimation + while (ddc->fft_sizepre_decimation) ddc->fft_size*=2; //fft_size should be a multiple of pre_decimation. ddc->overlap_length = ddc->taps_length - 1; ddc->input_size = ddc->fft_size - ddc->overlap_length; ddc->fft_inv_size = ddc->fft_size / ddc->pre_decimation; //Shift operation in the frequency domain: we can shift by a multiple of v. - ddc->v = ddc->fft_size/ddc->overlap_length; //+-1 ? (or maybe ceil() this?) //TODO: why? + ddc->v = ddc->fft_size/ddc->overlap_length; //overlap factor | +-1 ? (or maybe ceil() this?) int middlebin=ddc->fft_size / 2; ddc->startbin = middlebin + middlebin * shift_rate * 2; //fprintf(stderr, "ddc->startbin=%g\n",(float)ddc->startbin); @@ -64,7 +64,7 @@ int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shif ddc->pre_shift = ddc->offsetbin/(float)ddc->fft_size; ddc->dsadata = decimating_shift_addition_init(ddc->post_shift, ddc->post_decimation); - //Overlap is scrapd, not added + //Overlap is scrapped, not added ddc->scrap=ddc->overlap_length/ddc->pre_decimation; //TODO this is problematic sometimes! overlap_length = 401 :: scrap = 200 ddc->post_input_size=ddc->fft_inv_size-ddc->scrap; @@ -76,13 +76,13 @@ void fastddc_print(fastddc_t* ddc, char* source) { fprintf(stderr, "%s: fastddc_print_sizes(): (fft_size = %d) = (taps_length = %d) + (input_size = %d) - 1\n" - " overlap :: (overlap_length = %d) = taps_length - 1, taps_real_length = %d\n" + " overlap :: (overlap_length = %d) = taps_length - 1, taps_min_length = %d\n" " decimation :: decimation = (pre_decimation = %d) * (post_decimation = %d), fft_inv_size = %d\n" " shift :: startbin = %d, offsetbin = %d, v = %d, pre_shift = %g, post_shift = %g\n" " o&s :: post_input_size = %d, scrap = %d\n" , source, ddc->fft_size, ddc->taps_length, ddc->input_size, - ddc->overlap_length, ddc->taps_real_length, + ddc->overlap_length, ddc->taps_min_length, ddc->pre_decimation, ddc->post_decimation, ddc->fft_inv_size, ddc->startbin, ddc->offsetbin, ddc->v, ddc->pre_shift, ddc->post_shift, ddc->post_input_size, ddc->scrap ); @@ -120,16 +120,35 @@ decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* out } //Alias & shift & filter at once - fft_swap_sides(input, ddc->fft_size); //this is not very optimal, but now we stick with this slow solution until we got the algorithm working + fft_swap_sides(input, ddc->fft_size); //TODO this is not very optimal, but now we stick with this slow solution until we got the algorithm working + //fprintf(stderr, " === fastddc_inv_cc() ===\n"); for(int i=0;ifft_size;i++) { - int output_index = (ddc->startbin+i)%plan_inverse->size; - int tap_index = (ddc->fft_size+i-ddc->offsetbin)%ddc->fft_size; - cmultadd(inv_input+output_index, input+i, taps_fft+tap_index); //cmultadd(output, input1, input2): complex output += complex input1 * complex input 2 + int output_index = (ddc->fft_size+i-ddc->offsetbin)%plan_inverse->size; + int tap_index = i; + //fprintf(stderr, "output_index = %d , tap_index = %d, input index = %d\n", output_index, tap_index, i); + //cmultadd(inv_input+output_index, input+i, taps_fft+tap_index); //cmultadd(output, input1, input2): complex output += complex input1 * complex input 2 + // (a+b*i)*(c+d*i) = (ac-bd)+(ad+bc)*i + // a = iof(input,i) + // b = qof(input,i) + // c = iof(taps_fft,i) + // d = qof(taps_fft,i) + //iof(inv_input,output_index) += iof(input,i) * iof(taps_fft,i) - qof(input,i) * qof(taps_fft,i); + //qof(inv_input,output_index) += iof(input,i) * qof(taps_fft,i) + qof(input,i) * iof(taps_fft,i); + iof(inv_input,output_index) += iof(input,i); //no filter + qof(inv_input,output_index) += qof(input,i); } + //Normalize inv fft bins (now our output level is not higher than the input... but we may optimize this into the later loop when we normalize by size) + for(int i=0;isize;i++) + { + iof(inv_input,i)/=ddc->pre_decimation; + qof(inv_input,i)/=ddc->pre_decimation; + } + + fft_swap_sides(inv_input,plan_inverse->size); fft_execute(plan_inverse); - fft_swap_sides(inv_output,plan_inverse->size); + //Normalize data for(int i=0;isize;i++) //@fastddc_inv_cc: normalize by size @@ -140,7 +159,8 @@ decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* out //Overlap is scrapped, not added //Shift correction - shift_stat=decimating_shift_addition_cc(inv_output+ddc->scrap, output, ddc->post_input_size, ddc->dsadata, ddc->post_decimation, shift_stat); - //memcpy(inv_output+ddc->scrap, output + //shift_stat=decimating_shift_addition_cc(inv_output+ddc->scrap, output, ddc->post_input_size, ddc->dsadata, ddc->post_decimation, shift_stat); + shift_stat.output_size = ddc->post_input_size; //bypass shift correction + memcpy(output, inv_output+ddc->scrap, sizeof(complexf)*ddc->post_input_size); return shift_stat; } diff --git a/fastddc.h b/fastddc.h index d6e15d1..a948083 100644 --- a/fastddc.h +++ b/fastddc.h @@ -7,7 +7,7 @@ typedef struct fastddc_s int pre_decimation; int post_decimation; int taps_length; - int taps_real_length; + int taps_min_length; int overlap_length; //it is taps_length - 1 int fft_size; int fft_inv_size; From 6efdf7c8097eddbd0b0437043cbd4a4b4260fd21 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sun, 22 Nov 2015 15:43:52 +0100 Subject: [PATCH 15/74] Enabled the filter, the FFT of which also had to be swapped. Fixed the shift direction. --- csdr.c | 7 ++-- fastddc.c | 8 ++--- fastddc.h | 1 + grc_tests/test_fastddc.grc | 65 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 72 insertions(+), 9 deletions(-) diff --git a/csdr.c b/csdr.c index c7ada5b..18472d6 100644 --- a/csdr.c +++ b/csdr.c @@ -1725,9 +1725,10 @@ int main(int argc, char *argv[]) //make the filter float filter_half_bw = 0.5/decimation; - fprintf(stderr, "fastddc_inv_cc: preparing a bandpass filter of [%g, %g] cutoff rates. Real transition bandwidth is: %g\n", shift_rate-filter_half_bw, shift_rate+filter_half_bw, 4.0/ddc.taps_length); - firdes_bandpass_c(taps, ddc.taps_length, shift_rate-filter_half_bw, shift_rate+filter_half_bw, window); + fprintf(stderr, "fastddc_inv_cc: preparing a bandpass filter of [%g, %g] cutoff rates. Real transition bandwidth is: %g\n", (-shift_rate)-filter_half_bw, (-shift_rate)+filter_half_bw, 4.0/ddc.taps_length); + firdes_bandpass_c(taps, ddc.taps_length, (-shift_rate)-filter_half_bw, (-shift_rate)+filter_half_bw, window); fft_execute(plan_taps); + fft_swap_sides(taps_fft,ddc.fft_size); //make FFT plan complexf* inv_input = (complexf*)fft_malloc(sizeof(complexf)*ddc.fft_inv_size); @@ -1748,7 +1749,7 @@ int main(int argc, char *argv[]) fread(input, sizeof(complexf), ddc.fft_size, stdin); shift_stat = fastddc_inv_cc(input, output, &ddc, plan_inverse, taps_fft, shift_stat); fwrite(output, sizeof(complexf), shift_stat.output_size, stdout); - fprintf(stderr, "ss os = %d\n", shift_stat.output_size); + //fprintf(stderr, "ss os = %d\n", shift_stat.output_size); TRY_YIELD; } } diff --git a/fastddc.c b/fastddc.c index 9aab721..1e41043 100644 --- a/fastddc.c +++ b/fastddc.c @@ -133,10 +133,10 @@ decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* out // b = qof(input,i) // c = iof(taps_fft,i) // d = qof(taps_fft,i) - //iof(inv_input,output_index) += iof(input,i) * iof(taps_fft,i) - qof(input,i) * qof(taps_fft,i); - //qof(inv_input,output_index) += iof(input,i) * qof(taps_fft,i) + qof(input,i) * iof(taps_fft,i); - iof(inv_input,output_index) += iof(input,i); //no filter - qof(inv_input,output_index) += qof(input,i); + iof(inv_input,output_index) += iof(input,i) * iof(taps_fft,i) - qof(input,i) * qof(taps_fft,i); + qof(inv_input,output_index) += iof(input,i) * qof(taps_fft,i) + qof(input,i) * iof(taps_fft,i); + //iof(inv_input,output_index) += iof(input,i); //no filter + //qof(inv_input,output_index) += qof(input,i); } //Normalize inv fft bins (now our output level is not higher than the input... but we may optimize this into the later loop when we normalize by size) diff --git a/fastddc.h b/fastddc.h index a948083..d65ca3c 100644 --- a/fastddc.h +++ b/fastddc.h @@ -26,3 +26,4 @@ typedef struct fastddc_s int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shift_rate); decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* output, fastddc_t* ddc, FFT_PLAN_T* plan_inverse, complexf* taps_fft, decimating_shift_addition_status_t shift_stat); void fastddc_print(fastddc_t* ddc, char* source); +void fft_swap_sides(complexf* io, int fft_size); diff --git a/grc_tests/test_fastddc.grc b/grc_tests/test_fastddc.grc index 099e75b..492fcab 100644 --- a/grc_tests/test_fastddc.grc +++ b/grc_tests/test_fastddc.grc @@ -182,6 +182,61 @@ 250000 + + analog_noise_source_x + + amp + 1 + + + alias + + + + comment + + + + affinity + + + + _enabled + 0 + + + _coordinate + (224, 403) + + + _rotation + 0 + + + id + analog_noise_source_x_0 + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + noise_type + analog.GR_GAUSSIAN + + + type + complex + + + seed + 0 + + analog_sig_source_x @@ -202,7 +257,7 @@ _enabled - True + 1 freq @@ -308,7 +363,7 @@ commandline - csdr fastddc_fwd_cc 4 | csdr fastddc_inv_cc 4 0.1 + csdr fastddc_fwd_cc 4 | csdr fastddc_inv_cc 4 -0.1 comment @@ -683,6 +738,12 @@ Counts + + analog_noise_source_x_0 + blocks_throttle_0 + 0 + 0 + analog_sig_source_x_0 blocks_throttle_0 From 78791919352a3aa2e4791376220821261dc61fbe Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 24 Nov 2015 22:29:56 +0100 Subject: [PATCH 16/74] Fixed FastDDC! Now it works. --- fastddc.c | 14 +-- grc_tests/test_fastddc.grc | 223 ++++++++++++++++++++++++++++++++++++- 2 files changed, 228 insertions(+), 9 deletions(-) diff --git a/fastddc.c b/fastddc.c index 1e41043..7b26c0d 100644 --- a/fastddc.c +++ b/fastddc.c @@ -55,12 +55,12 @@ int fastddc_init(fastddc_t* ddc, float transition_bw, int decimation, float shif //Shift operation in the frequency domain: we can shift by a multiple of v. ddc->v = ddc->fft_size/ddc->overlap_length; //overlap factor | +-1 ? (or maybe ceil() this?) int middlebin=ddc->fft_size / 2; - ddc->startbin = middlebin + middlebin * shift_rate * 2; + ddc->startbin = middlebin + middlebin * (-shift_rate) * 2; //fprintf(stderr, "ddc->startbin=%g\n",(float)ddc->startbin); ddc->startbin = ddc->v * round( ddc->startbin / (float)ddc->v ); //fprintf(stderr, "ddc->startbin=%g\n",(float)ddc->startbin); ddc->offsetbin = ddc->startbin - middlebin; - ddc->post_shift = shift_rate-((float)ddc->offsetbin/ddc->fft_size); + ddc->post_shift = (ddc->pre_decimation)*(shift_rate+((float)ddc->offsetbin/ddc->fft_size)); ddc->pre_shift = ddc->offsetbin/(float)ddc->fft_size; ddc->dsadata = decimating_shift_addition_init(ddc->post_shift, ddc->post_decimation); @@ -122,9 +122,10 @@ decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* out //Alias & shift & filter at once fft_swap_sides(input, ddc->fft_size); //TODO this is not very optimal, but now we stick with this slow solution until we got the algorithm working //fprintf(stderr, " === fastddc_inv_cc() ===\n"); + //The problem is, we have to say that the output_index should be the _center_ of the spectrum when i is at startbin! (startbin is at the _center_ of the input to downconvert, not at its first bin!) for(int i=0;ifft_size;i++) { - int output_index = (ddc->fft_size+i-ddc->offsetbin)%plan_inverse->size; + int output_index = (ddc->fft_size+i-ddc->offsetbin+(ddc->fft_inv_size/2))%plan_inverse->size; int tap_index = i; //fprintf(stderr, "output_index = %d , tap_index = %d, input index = %d\n", output_index, tap_index, i); //cmultadd(inv_input+output_index, input+i, taps_fft+tap_index); //cmultadd(output, input1, input2): complex output += complex input1 * complex input 2 @@ -148,7 +149,6 @@ decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* out fft_swap_sides(inv_input,plan_inverse->size); fft_execute(plan_inverse); - //Normalize data for(int i=0;isize;i++) //@fastddc_inv_cc: normalize by size @@ -159,8 +159,8 @@ decimating_shift_addition_status_t fastddc_inv_cc(complexf* input, complexf* out //Overlap is scrapped, not added //Shift correction - //shift_stat=decimating_shift_addition_cc(inv_output+ddc->scrap, output, ddc->post_input_size, ddc->dsadata, ddc->post_decimation, shift_stat); - shift_stat.output_size = ddc->post_input_size; //bypass shift correction - memcpy(output, inv_output+ddc->scrap, sizeof(complexf)*ddc->post_input_size); + shift_stat=decimating_shift_addition_cc(inv_output+ddc->scrap, output, ddc->post_input_size, ddc->dsadata, ddc->post_decimation, shift_stat); + //shift_stat.output_size = ddc->post_input_size; //bypass shift correction + //memcpy(output, inv_output+ddc->scrap, sizeof(complexf)*ddc->post_input_size); return shift_stat; } diff --git a/grc_tests/test_fastddc.grc b/grc_tests/test_fastddc.grc index 492fcab..829a336 100644 --- a/grc_tests/test_fastddc.grc +++ b/grc_tests/test_fastddc.grc @@ -179,7 +179,7 @@ value - 250000 + 400000 @@ -237,6 +237,57 @@ 0 + + analog_pll_freqdet_cf + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (112, 875) + + + _rotation + 0 + + + id + analog_pll_freqdet_cf_0 + + + w + (3.141592654/200)/2 + + + max_freq + 3.141592654 + + + maxoutbuf + 0 + + + min_freq + -3.141592654 + + + minoutbuf + 0 + + analog_sig_source_x @@ -300,6 +351,57 @@ analog.GR_COS_WAVE + + blocks_multiply_const_vxx + + alias + + + + comment + + + + const + (samp_rate/decimation)*(1/(2*3.141592654)) + + + affinity + + + + _enabled + True + + + _coordinate + (136, 763) + + + _rotation + 0 + + + id + blocks_multiply_const_vxx_0 + + + type + float + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + vlen + 1 + + blocks_throttle @@ -363,7 +465,7 @@ commandline - csdr fastddc_fwd_cc 4 | csdr fastddc_inv_cc 4 -0.1 + csdr fastddc_fwd_cc %d | csdr fastddc_inv_cc %d 0.4"%(decimation,decimation)+" comment @@ -651,6 +753,105 @@ 10 + + wxgui_numbersink2 + + avg_alpha + 0 + + + average + False + + + alias + + + + comment + + + + affinity + + + + decimal_places + 10 + + + _enabled + True + + + factor + 1.0 + + + _coordinate + (400, 691) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_numbersink2_0 + + + max_value + (samp_rate/decimation)/2 + + + min_value + (-samp_rate/decimation)/2 + + + notebook + + + + number_rate + 15 + + + peak_hold + False + + + ref_level + 0 + + + samp_rate + samp_rate + + + show_gauge + True + + + title + PLL locked at + + + type + float + + + units + Hz + + + win_size + + + wxgui_scopesink2 @@ -744,12 +945,24 @@ 0 0 + + analog_pll_freqdet_cf_0 + blocks_multiply_const_vxx_0 + 0 + 0 + analog_sig_source_x_0 blocks_throttle_0 0 0 + + blocks_multiply_const_vxx_0 + wxgui_numbersink2_0 + 0 + 0 + blocks_throttle_0 ha5kfu_execproc_xx_1 @@ -762,6 +975,12 @@ 0 0 + + ha5kfu_execproc_xx_1 + analog_pll_freqdet_cf_0 + 0 + 0 + ha5kfu_execproc_xx_1 wxgui_fftsink2_0 From 4697928c38f37a4222081343c20b92a59275045c Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sun, 29 Nov 2015 12:54:09 +0000 Subject: [PATCH 17/74] Added shift_addfast_cc and tests. --- csdr.c | 49 ++ grc_tests/test_shift_remote.grc | 912 ++++++++++++++++++++++++++++++++ grc_tests/test_shift_remote.sh | 9 + libcsdr.c | 39 ++ libcsdr.h | 9 + make_test200 | 2 + test200.c | 84 +++ 7 files changed, 1104 insertions(+) create mode 100644 grc_tests/test_shift_remote.grc create mode 100755 grc_tests/test_shift_remote.sh create mode 100755 make_test200 create mode 100644 test200.c diff --git a/csdr.c b/csdr.c index 18472d6..bd74cf2 100644 --- a/csdr.c +++ b/csdr.c @@ -486,6 +486,55 @@ int main(int argc, char *argv[]) return 0; } + if(!strcmp(argv[1],"shift_addfast_cc")) + { + bigbufs=1; + + float starting_phase=0; + float rate; + + int fd; + if(fd=init_fifo(argc,argv)) + { + while(!read_fifo_ctl(fd,"%g\n",&rate)) usleep(10000); + } + else + { + if(argc<=2) return badsyntax("need required parameter (rate)"); + sscanf(argv[2],"%g",&rate); + } + + if(!sendbufsize(initialize_buffers())) return -2; + for(;;) + { + shift_addfast_data_t data=shift_addfast_init(rate); + fprintf(stderr,"shift_addfast_cc: reinitialized to %g\n",rate); + int remain, current_size; + float* ibufptr; + float* obufptr; + for(;;) + { + FEOF_CHECK; + if(!FREAD_C) break; + remain=the_bufsize; + ibufptr=input_buffer; + obufptr=output_buffer; + while(remain) + { + current_size=(remain>1024)?1024:remain; + starting_phase=shift_addfast_cc((complexf*)ibufptr, (complexf*)obufptr, current_size, &data, starting_phase); + ibufptr+=current_size*2; + obufptr+=current_size*2; + remain-=current_size; + } + FWRITE_C; + if(read_fifo_ctl(fd,"%g\n",&rate)) break; + TRY_YIELD; + } + } + return 0; + } + #ifdef LIBCSDR_GPL if(!strcmp(argv[1],"decimating_shift_addition_cc")) { diff --git a/grc_tests/test_shift_remote.grc b/grc_tests/test_shift_remote.grc new file mode 100644 index 0000000..516635f --- /dev/null +++ b/grc_tests/test_shift_remote.grc @@ -0,0 +1,912 @@ + + + + Thu Jan 15 18:51:48 2015 + + options + + author + + + + window_size + 1280, 1024 + + + category + Custom + + + comment + + + + description + + + + _enabled + True + + + _coordinate + (10, 10) + + + _rotation + 0 + + + generate_options + wx_gui + + + id + top_block + + + max_nouts + 0 + + + realtime_scheduling + + + + run_options + prompt + + + run + True + + + thread_safe_setters + + + + title + + + + + variable + + comment + + + + _enabled + True + + + _coordinate + (8, 195) + + + _rotation + 0 + + + id + rate + + + value + -0.1 + + + + variable + + comment + + + + _enabled + True + + + _coordinate + (176, 11) + + + _rotation + 0 + + + id + samp_rate + + + value + 250e3 + + + + analog_sig_source_x + + amp + 1 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + freq + 20000 + + + _coordinate + (8, 75) + + + _rotation + 0 + + + id + analog_sig_source_x_0 + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + offset + 0 + + + type + complex + + + samp_rate + samp_rate + + + waveform + analog.GR_CONST_WAVE + + + + blocks_throttle + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (224, 107) + + + _rotation + 0 + + + id + blocks_throttle_0 + + + ignoretag + True + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + samples_per_second + samp_rate + + + type + complex + + + vlen + 1 + + + + ha5kfu_execproc_xx + + alias + + + + commandline + "csdr shift_addition_cc %g"%rate + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (824, 315) + + + _rotation + 0 + + + id + ha5kfu_execproc_xx_0_0 + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + type + cc + + + + ha5kfu_execproc_xx + + alias + + + + commandline + ncat -vv raspberrypi.local 5321 + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (536, 443) + + + _rotation + 0 + + + id + ha5kfu_execproc_xx_0_0_0_1 + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + type + cc + + + + notebook + + alias + + + + comment + + + + _enabled + True + + + _coordinate + (272, 11) + + + _rotation + 0 + + + grid_pos + + + + id + nb0 + + + labels + ['original', 'shift_addition_cc','shift_addfast_cc',] + + + notebook + + + + style + wx.NB_TOP + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (848, 27) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0_0 + + + notebook + nb0,0 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT Plot + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (1112, 339) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0_1 + + + notebook + nb0,1 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT Plot + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (808, 387) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0_1_1 + + + notebook + nb0,2 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT Plot + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (1112, 555) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_scopesink2_0_0 + + + notebook + nb0,1 + + + num_inputs + 1 + + + samp_rate + samp_rate + + + t_scale + 0 + + + title + Scope Plot + + + trig_mode + wxgui.TRIG_MODE_NORM + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + + + + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (808, 611) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_scopesink2_0_0_1 + + + notebook + nb0,2 + + + num_inputs + 1 + + + samp_rate + samp_rate + + + t_scale + 0 + + + title + Scope Plot + + + trig_mode + wxgui.TRIG_MODE_NORM + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + + + + analog_sig_source_x_0 + blocks_throttle_0 + 0 + 0 + + + blocks_throttle_0 + ha5kfu_execproc_xx_0_0 + 0 + 0 + + + blocks_throttle_0 + ha5kfu_execproc_xx_0_0_0_1 + 0 + 0 + + + blocks_throttle_0 + wxgui_fftsink2_0_0 + 0 + 0 + + + ha5kfu_execproc_xx_0_0 + wxgui_fftsink2_0_1 + 0 + 0 + + + ha5kfu_execproc_xx_0_0 + wxgui_scopesink2_0_0 + 0 + 0 + + + ha5kfu_execproc_xx_0_0_0_1 + wxgui_fftsink2_0_1_1 + 0 + 0 + + + ha5kfu_execproc_xx_0_0_0_1 + wxgui_scopesink2_0_0_1 + 0 + 0 + + diff --git a/grc_tests/test_shift_remote.sh b/grc_tests/test_shift_remote.sh new file mode 100755 index 0000000..65c7192 --- /dev/null +++ b/grc_tests/test_shift_remote.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Run this script on a Raspberry Pi 2, while running test_shift_remote.grc on your PC. +# It allows you to debug the NEON-accelerated version of specific DSP algorithms on the target hardware. +TEMPSCRIPT="/tmp/test_shift_remote_exec.sh" +echo '#!/bin/sh\ncsdr shift_addfast_cc -0.1' > $TEMPSCRIPT +cat $TEMPSCRIPT +chmod +x $TEMPSCRIPT +ncat -vvl 5321 -e $TEMPSCRIPT +rm $TEMPSCRIPT diff --git a/libcsdr.c b/libcsdr.c index e6b5b31..09fe664 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -263,6 +263,45 @@ float shift_table_cc(complexf* input, complexf* output, int input_size, float ra return phase; } + + +shift_addfast_data_t shift_addfast_init(float rate) +{ + shift_addfast_data_t output; + float phase_increment=2*rate*PI; + for(int i=0;i<4;i++) + { + output.dsin[i]=sin(phase_increment*(i+1)); + output.dcos[i]=cos(phase_increment*(i+1)); + } + return output; +} + +float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_addfast_data_t* d, float starting_phase) +{ + //input_size should be multiple of 4 + float phase=starting_phase; + float cos_start=cos(starting_phase); + float sin_start=sin(starting_phase); + float cos_vals[4], sin_vals[4]; + for(int i=0;idcos[i] - sin_start * d->dsin[i]; + sin_vals[i] = sin_start * d->dcos[i] + cos_start * d->dsin[i]; + } + for(int j=0;j<4;j++) + { + iof(output,4*i+j)=cos_vals[j]*iof(input,4*i+j)-sin_vals[j]*qof(input,4*i+j); + qof(output,4*i+j)=sin_vals[j]*iof(input,4*i+j)+cos_vals[j]*qof(input,4*i+j); + } + cos_start = cos_vals[3]; + sin_start = sin_vals[3]; + } + return phase; +} + #ifdef NEON_OPTS #pragma message "We have a faster fir_decimate_cc now." diff --git a/libcsdr.h b/libcsdr.h index ca4a311..f3e154e 100644 --- a/libcsdr.h +++ b/libcsdr.h @@ -156,6 +156,15 @@ void shift_table_deinit(shift_table_data_t table_data); shift_table_data_t shift_table_init(int table_size); float shift_table_cc(complexf* input, complexf* output, int input_size, float rate, shift_table_data_t table_data, float starting_phase); +typedef struct shift_addfast_data_s +{ + float dsin[4]; + float dcos[4]; + +} shift_addfast_data_t; +shift_addfast_data_t shift_addfast_init(float rate); +float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_addfast_data_t* d, float starting_phase); + int log2n(int x); int next_pow2(int x); diff --git a/make_test200 b/make_test200 new file mode 100755 index 0000000..7e5cc05 --- /dev/null +++ b/make_test200 @@ -0,0 +1,2 @@ +#!/bin/bash +gcc test200.c --std=gnu99 -o test200 -DUSE_FFTW -DLIBCSDR_GPL -lcsdr diff --git a/test200.c b/test200.c new file mode 100644 index 0000000..c2166c9 --- /dev/null +++ b/test200.c @@ -0,0 +1,84 @@ +/* +This software is part of libcsdr, a set of simple DSP routines for +Software Defined Radio. + +Copyright (c) 2014-2015, Andras Retzler +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL ANDRAS RETZLER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "libcsdr.h" +#include "libcsdr_gpl.h" + +#define T_BUFSIZE (1024*1024/4) +#define T_N (200) + +int main() +{ + fprintf(stderr,"Getting a %d of random samples...\n", T_BUFSIZE); + int urand_fp = open("/dev/urandom",O_RDWR); + unsigned char* buf_u8 = (unsigned char*)malloc(sizeof(unsigned char)*T_BUFSIZE*2); + complexf* buf_c = (complexf*)malloc(sizeof(complexf)*T_BUFSIZE); + complexf* outbuf_c = (complexf*)malloc(sizeof(complexf)*T_BUFSIZE); + read(urand_fp, buf_u8, T_BUFSIZE); + close(urand_fp); + + for(int i=0;i Date: Sun, 29 Nov 2015 19:05:28 +0000 Subject: [PATCH 18/74] Added NEON implementation of shift_addfast_cc. --- csdr.c | 2 -- libcsdr.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++----- libcsdr.h | 4 ++- test200.c | 2 +- 4 files changed, 93 insertions(+), 13 deletions(-) diff --git a/csdr.c b/csdr.c index bd74cf2..a1330e3 100644 --- a/csdr.c +++ b/csdr.c @@ -1274,8 +1274,6 @@ int main(int argc, char *argv[]) } #endif -#define TIME_TAKEN(start,end) ((end.tv_sec-start.tv_sec)+(end.tv_nsec-start.tv_nsec)/1e9) - if(!strcmp(argv[1],"fft_benchmark")) { if(argc<=3) return badsyntax("need required parameters (fft_size, fft_cycles)"); diff --git a/libcsdr.c b/libcsdr.c index 09fe664..e3c8d33 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -268,30 +268,108 @@ float shift_table_cc(complexf* input, complexf* output, int input_size, float ra shift_addfast_data_t shift_addfast_init(float rate) { shift_addfast_data_t output; - float phase_increment=2*rate*PI; + output.phase_increment=2*rate*PI; for(int i=0;i<4;i++) { - output.dsin[i]=sin(phase_increment*(i+1)); - output.dcos[i]=cos(phase_increment*(i+1)); + output.dsin[i]=sin(output.phase_increment*(i+1)); + output.dcos[i]=cos(output.phase_increment*(i+1)); } return output; } +#ifdef NEON_OPTS +#pragma message "Manual NEON optimizations are ON: we have a faster shift_addfast_cc now." + float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_addfast_data_t* d, float starting_phase) { //input_size should be multiple of 4 float phase=starting_phase; + float cos_start[4], sin_start[4]; + float cos_vals[4], sin_vals[4]; + for(int i=0;i<4;i++) + { + cos_start[i] = cos(starting_phase); + sin_start[i] = sin(starting_phase); + } + + float* pdcos = d->dcos; + float* pdsin = d->dsin; + register float* pinput = (float*)input; + register float* pinput_end = ((float*)input)+input_size; + register float* poutput = (float*)output; + + #define RDCOS "q0" //dcos, dsin + #define RDSIN "q1" + #define RCOSST "q2" //cos_start, sin_start + #define RSINST "q3" + #define RCOSV "q4" //cos_vals, sin_vals + #define RSINV "q5" + #define ROUTI "q6" //output_i, output_q + #define ROUTQ "q7" + #define RINPI "q8" //input_i, input_q + #define RINPQ "q9" + #define R3(x,y,z) x ", " y ", " z "\n\t" + + asm volatile( //(the range is q0-015) + " vld1.32 {" RDCOS "}, [%[pdcos]]\n\t" + " vld1.32 {" RDSIN "}, [%[pdsin]]\n\t" + " vld1.32 {" RCOSST "}, [%[cos_start]]\n\t" + " vld1.32 {" RSINST "}, [%[sin_start]]\n\t" + "for_addfast: vld2.32 {" RINPI "-" RINPQ "}, [%[pinput]]!\n\t" //load q0 and q1 directly from the memory address stored in pinput, with interleaving (so that we get the I samples in rinpi and the Q samples in rinpq), also increment the memory address in pinput (hence the "!" mark) + + //C version: + //cos_vals[j] = cos_start * d->dcos[j] - sin_start * d->dsin[j]; + //sin_vals[j] = sin_start * d->dcos[j] + cos_start * d->dsin[j]; + + " vmul.f32 " R3(RCOSV, RCOSST, RDCOS) //cos_vals[i] = cos_start * d->dcos[i] + " vmls.f32 " R3(RCOSV, RSINST, RDSIN) //cos_vals[i] -= sin_start * d->dsin[i] + " vmul.f32 " R3 (RSINV, RSINST, RDCOS) //sin_vals[i] = sin_start * d->dcos[i] + " vmla.f32 " R3(RCOSV, RSINST, RDSIN) //sin_vals[i] += cos_start * d->dsin[i] + + //C version: + //iof(output,4*i+j)=cos_vals[j]*iof(input,4*i+j)-sin_vals[j]*qof(input,4*i+j); + //qof(output,4*i+j)=sin_vals[j]*iof(input,4*i+j)+cos_vals[j]*qof(input,4*i+j); + " vmul.f32 " R3(ROUTI, RCOSV, RINPI) //output = cos_vals * input + " vmls.f32 " R3(ROUTI, RSINV, RINPQ) //output -= sin_vals * input + " vmul.f32 " R3(ROUTQ, RSINV, RINPI) //sin_vals[i] = sin_start * d->dcos[i] + " vmla.f32 " R3(ROUTQ, RCOSV, RINPQ) //sin_vals[i] += cos_start * d->dsin[i] + + " vst2.32 {" ROUTI "-" ROUTQ "}, [%[poutput]]!\n\t" //store the outputs in memory + + " vdup.32 " RCOSST ", d5[1]\n\t" // cos_start[0-3] = cos_vals[3] + " vdup.32 " RSINST ", d7[1]\n\t" // sin_start[0-3] = sin_vals[3] + + " cmp %[pinput], %[pinput_end]\n\t" //if(pinput == pinput_end) + " bcc for_fdccasm\n\t" // then goto for_fdcasm + : + [pinput]"+r"(pinput), [poutput]"+r"(poutput) //output operand list -> C variables that we will change from ASM + : + [pinput_end]"r"(pinput_end), [pdcos]"r"(pdcos), [pdsin]"r"(pdsin), [sin_start]"r"(sin_start), [cos_start]"r"(cos_start) //input operand list + : + "memory", "q0", "q1", "q2", "q4", "q5", "q6", "q7", "q8", "q9", "cc" //clobber list + ); + + return phase+input_size*d->phase_increment; +} + +#else + +float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_addfast_data_t* d, float starting_phase) +{ + //input_size should be multiple of 4 + //fprintf(stderr, "shift_addfast_cc: input_size = %d\n", input_size); + float phase=starting_phase; float cos_start=cos(starting_phase); float sin_start=sin(starting_phase); float cos_vals[4], sin_vals[4]; for(int i=0;idcos[i] - sin_start * d->dsin[i]; - sin_vals[i] = sin_start * d->dcos[i] + cos_start * d->dsin[i]; + cos_vals[j] = cos_start * d->dcos[j] - sin_start * d->dsin[j]; + sin_vals[j] = sin_start * d->dcos[j] + cos_start * d->dsin[j]; } - for(int j=0;j<4;j++) + for(int j=0;j<4;j++) //@shift_addfast_cc { iof(output,4*i+j)=cos_vals[j]*iof(input,4*i+j)-sin_vals[j]*qof(input,4*i+j); qof(output,4*i+j)=sin_vals[j]*iof(input,4*i+j)+cos_vals[j]*qof(input,4*i+j); @@ -299,11 +377,13 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ cos_start = cos_vals[3]; sin_start = sin_vals[3]; } - return phase; + return phase+input_size*d->phase_increment; } +#endif + #ifdef NEON_OPTS -#pragma message "We have a faster fir_decimate_cc now." +#pragma message "Manual NEON optimizations are ON: we have a faster fir_decimate_cc now." //max help: http://community.arm.com/groups/android-community/blog/2015/03/27/arm-neon-programming-quick-reference diff --git a/libcsdr.h b/libcsdr.h index f3e154e..5ccb370 100644 --- a/libcsdr.h +++ b/libcsdr.h @@ -63,6 +63,8 @@ typedef struct complexf_s { float i; float q; } complexf; //they dropped M_PI in C99, so we define it: #define PI ((float)3.14159265358979323846) +#define TIME_TAKEN(start,end) ((end.tv_sec-start.tv_sec)+(end.tv_nsec-start.tv_nsec)/1e9) + //window typedef enum window_s { @@ -160,7 +162,7 @@ typedef struct shift_addfast_data_s { float dsin[4]; float dcos[4]; - + float phase_increment; } shift_addfast_data_t; shift_addfast_data_t shift_addfast_init(float rate); float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_addfast_data_t* d, float starting_phase); diff --git a/test200.c b/test200.c index c2166c9..f233d1a 100644 --- a/test200.c +++ b/test200.c @@ -78,7 +78,7 @@ int main() clock_gettime(CLOCK_MONOTONIC_RAW, &start_time); for(int i=0;i Date: Sun, 29 Nov 2015 19:13:24 +0000 Subject: [PATCH 19/74] Fixed a jump. --- libcsdr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libcsdr.c b/libcsdr.c index e3c8d33..64099bb 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -310,7 +310,7 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ #define RINPQ "q9" #define R3(x,y,z) x ", " y ", " z "\n\t" - asm volatile( //(the range is q0-015) + asm volatile( //(the range of q is q0-q15) " vld1.32 {" RDCOS "}, [%[pdcos]]\n\t" " vld1.32 {" RDSIN "}, [%[pdsin]]\n\t" " vld1.32 {" RCOSST "}, [%[cos_start]]\n\t" @@ -323,7 +323,7 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ " vmul.f32 " R3(RCOSV, RCOSST, RDCOS) //cos_vals[i] = cos_start * d->dcos[i] " vmls.f32 " R3(RCOSV, RSINST, RDSIN) //cos_vals[i] -= sin_start * d->dsin[i] - " vmul.f32 " R3 (RSINV, RSINST, RDCOS) //sin_vals[i] = sin_start * d->dcos[i] + " vmul.f32 " R3(RSINV, RSINST, RDCOS) //sin_vals[i] = sin_start * d->dcos[i] " vmla.f32 " R3(RCOSV, RSINST, RDSIN) //sin_vals[i] += cos_start * d->dsin[i] //C version: @@ -340,7 +340,7 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ " vdup.32 " RSINST ", d7[1]\n\t" // sin_start[0-3] = sin_vals[3] " cmp %[pinput], %[pinput_end]\n\t" //if(pinput == pinput_end) - " bcc for_fdccasm\n\t" // then goto for_fdcasm + " bcc for_addfast\n\t" // then goto for_fdcasm : [pinput]"+r"(pinput), [poutput]"+r"(poutput) //output operand list -> C variables that we will change from ASM : From 965ea631fcd46e208d5051bffc87351d29dcb659 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sun, 29 Nov 2015 20:47:00 +0000 Subject: [PATCH 20/74] Now at least we get a continuous output signal from shift_addfast_cc/NEON. And we get a good result if the shift_rate is 0! (Okay, we get a bad result for anything else...) --- grc_tests/test_shift_remote.grc | 63 +++++++++++++++++++++++++++++++-- grc_tests/test_shift_remote.sh | 2 +- libcsdr.c | 10 +++--- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/grc_tests/test_shift_remote.grc b/grc_tests/test_shift_remote.grc index 516635f..9b4f589 100644 --- a/grc_tests/test_shift_remote.grc +++ b/grc_tests/test_shift_remote.grc @@ -69,6 +69,65 @@ + + variable_slider + + comment + + + + converver + float_converter + + + value + 0 + + + _enabled + True + + + _coordinate + (24, 331) + + + _rotation + 0 + + + grid_pos + + + + id + gen_freq + + + label + Frequency: + + + max + samp_rate/2 + + + min + -samp_rate/2 + + + notebook + + + + num_steps + 100 + + + style + wx.SL_HORIZONTAL + + variable @@ -147,7 +206,7 @@ freq - 20000 + gen_freq _coordinate @@ -183,7 +242,7 @@ waveform - analog.GR_CONST_WAVE + analog.GR_SIN_WAVE diff --git a/grc_tests/test_shift_remote.sh b/grc_tests/test_shift_remote.sh index 65c7192..14f061d 100755 --- a/grc_tests/test_shift_remote.sh +++ b/grc_tests/test_shift_remote.sh @@ -2,7 +2,7 @@ # Run this script on a Raspberry Pi 2, while running test_shift_remote.grc on your PC. # It allows you to debug the NEON-accelerated version of specific DSP algorithms on the target hardware. TEMPSCRIPT="/tmp/test_shift_remote_exec.sh" -echo '#!/bin/sh\ncsdr shift_addfast_cc -0.1' > $TEMPSCRIPT +echo '#!/bin/sh\ncsdr shift_addfast_cc -0' > $TEMPSCRIPT cat $TEMPSCRIPT chmod +x $TEMPSCRIPT ncat -vvl 5321 -e $TEMPSCRIPT diff --git a/libcsdr.c b/libcsdr.c index 64099bb..d0c3b2d 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -295,9 +295,10 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ float* pdcos = d->dcos; float* pdsin = d->dsin; register float* pinput = (float*)input; - register float* pinput_end = ((float*)input)+input_size; + register float* pinput_end = (float*)(input+input_size); register float* poutput = (float*)output; + //Register map: #define RDCOS "q0" //dcos, dsin #define RDSIN "q1" #define RCOSST "q2" //cos_start, sin_start @@ -324,7 +325,7 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ " vmul.f32 " R3(RCOSV, RCOSST, RDCOS) //cos_vals[i] = cos_start * d->dcos[i] " vmls.f32 " R3(RCOSV, RSINST, RDSIN) //cos_vals[i] -= sin_start * d->dsin[i] " vmul.f32 " R3(RSINV, RSINST, RDCOS) //sin_vals[i] = sin_start * d->dcos[i] - " vmla.f32 " R3(RCOSV, RSINST, RDSIN) //sin_vals[i] += cos_start * d->dsin[i] + " vmla.f32 " R3(RSINV, RCOSST, RDSIN) //sin_vals[i] += cos_start * d->dsin[i] //C version: //iof(output,4*i+j)=cos_vals[j]*iof(input,4*i+j)-sin_vals[j]*qof(input,4*i+j); @@ -334,8 +335,8 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ " vmul.f32 " R3(ROUTQ, RSINV, RINPI) //sin_vals[i] = sin_start * d->dcos[i] " vmla.f32 " R3(ROUTQ, RCOSV, RINPQ) //sin_vals[i] += cos_start * d->dsin[i] - " vst2.32 {" ROUTI "-" ROUTQ "}, [%[poutput]]!\n\t" //store the outputs in memory - + " vst2.32 {" ROUTI "-" ROUTQ "}, [%[poutput]]\n\t" //store the outputs in memory + " add %[poutput],%[poutput],#32\n\t" " vdup.32 " RCOSST ", d5[1]\n\t" // cos_start[0-3] = cos_vals[3] " vdup.32 " RSINST ", d7[1]\n\t" // sin_start[0-3] = sin_vals[3] @@ -348,7 +349,6 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ : "memory", "q0", "q1", "q2", "q4", "q5", "q6", "q7", "q8", "q9", "cc" //clobber list ); - return phase+input_size*d->phase_increment; } From 95ebc0e7906c5895a601366c0d1b349688333afd Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sun, 29 Nov 2015 22:46:06 +0000 Subject: [PATCH 21/74] Fixed shift_addfast_cc/NEON, now it works! Added shift_unroll_cc. --- csdr.c | 50 ++++++++++++++++++++++ grc_tests/test_shift_remote.grc | 2 +- grc_tests/test_shift_remote.sh | 2 +- libcsdr.c | 74 ++++++++++++++++++++++++++------- libcsdr.h | 10 +++++ test200.c | 20 ++++++++- 6 files changed, 139 insertions(+), 19 deletions(-) diff --git a/csdr.c b/csdr.c index a1330e3..e74fadc 100644 --- a/csdr.c +++ b/csdr.c @@ -535,6 +535,56 @@ int main(int argc, char *argv[]) return 0; } + + if(!strcmp(argv[1],"shift_unroll_cc")) + { + bigbufs=1; + + float starting_phase=0; + float rate; + + int fd; + if(fd=init_fifo(argc,argv)) + { + while(!read_fifo_ctl(fd,"%g\n",&rate)) usleep(10000); + } + else + { + if(argc<=2) return badsyntax("need required parameter (rate)"); + sscanf(argv[2],"%g",&rate); + } + + if(!sendbufsize(initialize_buffers())) return -2; + for(;;) + { + shift_unroll_data_t data=shift_unroll_init(rate, 1024); + fprintf(stderr,"shift_unroll_cc: reinitialized to %g\n",rate); + int remain, current_size; + float* ibufptr; + float* obufptr; + for(;;) + { + FEOF_CHECK; + if(!FREAD_C) break; + remain=the_bufsize; + ibufptr=input_buffer; + obufptr=output_buffer; + while(remain) + { + current_size=(remain>1024)?1024:remain; + starting_phase=shift_unroll_cc((complexf*)ibufptr, (complexf*)obufptr, current_size, &data, starting_phase); + ibufptr+=current_size*2; + obufptr+=current_size*2; + remain-=current_size; + } + FWRITE_C; + if(read_fifo_ctl(fd,"%g\n",&rate)) break; + TRY_YIELD; + } + } + return 0; + } + #ifdef LIBCSDR_GPL if(!strcmp(argv[1],"decimating_shift_addition_cc")) { diff --git a/grc_tests/test_shift_remote.grc b/grc_tests/test_shift_remote.grc index 9b4f589..59620bd 100644 --- a/grc_tests/test_shift_remote.grc +++ b/grc_tests/test_shift_remote.grc @@ -355,7 +355,7 @@ commandline - ncat -vv raspberrypi.local 5321 + ncat -v raspberrypi.local 5321 comment diff --git a/grc_tests/test_shift_remote.sh b/grc_tests/test_shift_remote.sh index 14f061d..65c7192 100755 --- a/grc_tests/test_shift_remote.sh +++ b/grc_tests/test_shift_remote.sh @@ -2,7 +2,7 @@ # Run this script on a Raspberry Pi 2, while running test_shift_remote.grc on your PC. # It allows you to debug the NEON-accelerated version of specific DSP algorithms on the target hardware. TEMPSCRIPT="/tmp/test_shift_remote_exec.sh" -echo '#!/bin/sh\ncsdr shift_addfast_cc -0' > $TEMPSCRIPT +echo '#!/bin/sh\ncsdr shift_addfast_cc -0.1' > $TEMPSCRIPT cat $TEMPSCRIPT chmod +x $TEMPSCRIPT ncat -vvl 5321 -e $TEMPSCRIPT diff --git a/libcsdr.c b/libcsdr.c index d0c3b2d..df87266 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -264,6 +264,44 @@ float shift_table_cc(complexf* input, complexf* output, int input_size, float ra } +shift_unroll_data_t shift_unroll_init(float rate, int size) +{ + shift_unroll_data_t output; + output.phase_increment=2*rate*PI; + output.size = size; + output.dsin=(float*)malloc(sizeof(float)*size); + output.dcos=(float*)malloc(sizeof(float)*size); + float myphase = 0; + for(int i=0;iPI) myphase-=2*PI; + while(myphase<-PI) myphase+=2*PI; + output.dsin[i]=sin(myphase); + output.dcos[i]=cos(myphase); + } + return output; +} + +float shift_unroll_cc(complexf *input, complexf* output, int input_size, shift_unroll_data_t* d, float starting_phase) +{ + //input_size should be multiple of 4 + //fprintf(stderr, "shift_addfast_cc: input_size = %d\n", input_size); + float cos_start=cos(starting_phase); + float sin_start=sin(starting_phase); + register float cos_val, sin_val; + for(int i=0;idcos[i] - sin_start * d->dsin[i]; + sin_val = sin_start * d->dcos[i] + cos_start * d->dsin[i]; + iof(output,i)=cos_val*iof(input,i)-sin_val*qof(input,i); + qof(output,i)=sin_val*iof(input,i)+cos_val*qof(input,i); + } + starting_phase+=input_size*d->phase_increment; + while(starting_phase>PI) starting_phase-=2*PI; + while(starting_phase<-PI) starting_phase+=2*PI; + return starting_phase; +} shift_addfast_data_t shift_addfast_init(float rate) { @@ -283,7 +321,6 @@ shift_addfast_data_t shift_addfast_init(float rate) float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_addfast_data_t* d, float starting_phase) { //input_size should be multiple of 4 - float phase=starting_phase; float cos_start[4], sin_start[4]; float cos_vals[4], sin_vals[4]; for(int i=0;i<4;i++) @@ -316,7 +353,7 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ " vld1.32 {" RDSIN "}, [%[pdsin]]\n\t" " vld1.32 {" RCOSST "}, [%[cos_start]]\n\t" " vld1.32 {" RSINST "}, [%[sin_start]]\n\t" - "for_addfast: vld2.32 {" RINPI "-" RINPQ "}, [%[pinput]]!\n\t" //load q0 and q1 directly from the memory address stored in pinput, with interleaving (so that we get the I samples in rinpi and the Q samples in rinpq), also increment the memory address in pinput (hence the "!" mark) + "for_addfast: vld2.32 {" RINPI "-" RINPQ "}, [%[pinput]]!\n\t" //load q0 and q1 directly from the memory address stored in pinput, with interleaving (so that we get the I samples in RINPI and the Q samples in RINPQ), also increment the memory address in pinput (hence the "!" mark) //C version: //cos_vals[j] = cos_start * d->dcos[j] - sin_start * d->dsin[j]; @@ -330,18 +367,18 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ //C version: //iof(output,4*i+j)=cos_vals[j]*iof(input,4*i+j)-sin_vals[j]*qof(input,4*i+j); //qof(output,4*i+j)=sin_vals[j]*iof(input,4*i+j)+cos_vals[j]*qof(input,4*i+j); - " vmul.f32 " R3(ROUTI, RCOSV, RINPI) //output = cos_vals * input - " vmls.f32 " R3(ROUTI, RSINV, RINPQ) //output -= sin_vals * input - " vmul.f32 " R3(ROUTQ, RSINV, RINPI) //sin_vals[i] = sin_start * d->dcos[i] - " vmla.f32 " R3(ROUTQ, RCOSV, RINPQ) //sin_vals[i] += cos_start * d->dsin[i] + " vmul.f32 " R3(ROUTI, RCOSV, RINPI) //output_i = cos_vals * input_i + " vmls.f32 " R3(ROUTI, RSINV, RINPQ) //output_i -= sin_vals * input_q + " vmul.f32 " R3(ROUTQ, RSINV, RINPI) //output_q = sin_vals * input_i + " vmla.f32 " R3(ROUTQ, RCOSV, RINPQ) //output_i += cos_vals * input_q - " vst2.32 {" ROUTI "-" ROUTQ "}, [%[poutput]]\n\t" //store the outputs in memory - " add %[poutput],%[poutput],#32\n\t" - " vdup.32 " RCOSST ", d5[1]\n\t" // cos_start[0-3] = cos_vals[3] - " vdup.32 " RSINST ", d7[1]\n\t" // sin_start[0-3] = sin_vals[3] + " vst2.32 {" ROUTI "-" ROUTQ "}, [%[poutput]]!\n\t" //store the outputs in memory + //" add %[poutput],%[poutput],#32\n\t" + " vdup.32 " RCOSST ", d9[1]\n\t" // cos_start[0-3] = cos_vals[3] + " vdup.32 " RSINST ", d11[1]\n\t" // sin_start[0-3] = sin_vals[3] - " cmp %[pinput], %[pinput_end]\n\t" //if(pinput == pinput_end) - " bcc for_addfast\n\t" // then goto for_fdcasm + " cmp %[pinput], %[pinput_end]\n\t" //if(pinput != pinput_end) + " bcc for_addfast\n\t" // then goto for_addfast : [pinput]"+r"(pinput), [poutput]"+r"(poutput) //output operand list -> C variables that we will change from ASM : @@ -349,7 +386,10 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ : "memory", "q0", "q1", "q2", "q4", "q5", "q6", "q7", "q8", "q9", "cc" //clobber list ); - return phase+input_size*d->phase_increment; + starting_phase+=input_size*d->phase_increment; + while(starting_phase>PI) starting_phase-=2*PI; + while(starting_phase<-PI) starting_phase+=2*PI; + return starting_phase; } #else @@ -358,7 +398,6 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ { //input_size should be multiple of 4 //fprintf(stderr, "shift_addfast_cc: input_size = %d\n", input_size); - float phase=starting_phase; float cos_start=cos(starting_phase); float sin_start=sin(starting_phase); float cos_vals[4], sin_vals[4]; @@ -377,7 +416,10 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ cos_start = cos_vals[3]; sin_start = sin_vals[3]; } - return phase+input_size*d->phase_increment; + starting_phase+=input_size*d->phase_increment; + while(starting_phase>PI) starting_phase-=2*PI; + while(starting_phase<-PI) starting_phase+=2*PI; + return starting_phase; } #endif @@ -422,7 +464,7 @@ q4, q5: accumulator for I branch and Q branch (will be the output) " vld1.32 {q2}, [%[ptaps]]!\n\t" " vmla.f32 q4, q0, q2\n\t" //quad_acc_i += quad_input_i * quad_taps_1 //http://stackoverflow.com/questions/3240440/how-to-use-the-multiply-and-accumulate-intrinsics-in-arm-cortex-a8 //http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0489e/CIHEJBIE.html " vmla.f32 q5, q1, q2\n\t" //quad_acc_q += quad_input_q * quad_taps_1 - " cmp %[ptaps], %[ptaps_end]\n\t" //if(ptaps == ptaps_end) + " cmp %[ptaps], %[ptaps_end]\n\t" //if(ptaps != ptaps_end) " bcc for_fdccasm\n\t" // then goto for_fdcasm " vst1.32 {q4}, [%[quad_acci]]\n\t" //if the loop is finished, store the two accumulators in memory " vst1.32 {q5}, [%[quad_accq]]\n\t" diff --git a/libcsdr.h b/libcsdr.h index 5ccb370..334ba6f 100644 --- a/libcsdr.h +++ b/libcsdr.h @@ -165,8 +165,18 @@ typedef struct shift_addfast_data_s float phase_increment; } shift_addfast_data_t; shift_addfast_data_t shift_addfast_init(float rate); +shift_addfast_data_t shift_addfast_init(float rate); float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_addfast_data_t* d, float starting_phase); +typedef struct shift_unroll_data_s +{ + float* dsin; + float* dcos; + float phase_increment; + int size; +} shift_unroll_data_t; +float shift_unroll_cc(complexf *input, complexf* output, int input_size, shift_unroll_data_t* d, float starting_phase); +shift_unroll_data_t shift_unroll_init(float rate, int size); int log2n(int x); int next_pow2(int x); diff --git a/test200.c b/test200.c index f233d1a..9feb457 100644 --- a/test200.c +++ b/test200.c @@ -62,9 +62,18 @@ int main() fprintf(stderr,"Starting tests of processing %d samples...\n", T_BUFSIZE*T_N); + //shift_math_cc + float starting_phase = 0; + + clock_gettime(CLOCK_MONOTONIC_RAW, &start_time); + for(int i=0;i Date: Mon, 30 Nov 2015 00:20:22 +0100 Subject: [PATCH 22/74] Tested all shifts, and they seem to work! --- grc_tests/test_shift.grc | 2573 ++++++++++++++++++++++++-------------- libcsdr.c | 39 + test200.c | 9 + 3 files changed, 1653 insertions(+), 968 deletions(-) diff --git a/grc_tests/test_shift.grc b/grc_tests/test_shift.grc index 9fc1eb7..0b48387 100644 --- a/grc_tests/test_shift.grc +++ b/grc_tests/test_shift.grc @@ -1,49 +1,49 @@ - - + + Thu Jan 15 18:51:48 2015 options - - id - top_block - - - _enabled - True - - - title - - author - - description - - window_size 1280, 1024 - - generate_options - wx_gui - category Custom - run_options - prompt + comment + - run + description + + + + _enabled True + + _coordinate + (10, 10) + + + _rotation + 0 + + + generate_options + wx_gui + + + id + top_block + max_nouts 0 @@ -53,63 +53,32 @@ - alias + run_options + prompt + + + run + True + + + thread_safe_setters - _coordinate - (10, 10) - - - _rotation - 0 + title + variable - id - rate + comment + _enabled True - - value - -0.055 - - - alias - - - - _coordinate - (8, 195) - - - _rotation - 0 - - - - variable - - id - decimation - - - _enabled - True - - - value - 3 - - - alias - - _coordinate (16, 267) @@ -118,25 +87,111 @@ _rotation 0 - - - variable id - samp_rate + decimation + + + value + 3 + + + + variable_slider + + comment + + + + converver + float_converter + + + value + 0 _enabled True - value - 250e3 + _coordinate + (184, 219) - alias + _rotation + 0 + + + grid_pos + + id + gen_freq + + + label + + + + max + samp_rate/2 + + + min + -samp_rate/2 + + + notebook + + + + num_steps + 100 + + + style + wx.SL_HORIZONTAL + + + + variable + + comment + + + + _enabled + True + + + _coordinate + (8, 195) + + + _rotation + 0 + + + id + rate + + + value + -0.055 + + + + variable + + comment + + + + _enabled + True + _coordinate (176, 11) @@ -145,16 +200,64 @@ _rotation 0 + + id + samp_rate + + + value + 250e3 + analog_sig_source_x + + amp + 1 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + freq + gen_freq + + + _coordinate + (8, 67) + + + _rotation + 0 + id analog_sig_source_x_0 - _enabled - True + maxoutbuf + 0 + + + minoutbuf + 0 + + + offset + 0 type @@ -166,605 +269,217 @@ waveform - analog.GR_CONST_WAVE - - - freq - 20000 - - - amp - 1 - - - offset - 0 - - - alias - - - - affinity - - - - minoutbuf - 0 - - - maxoutbuf - 0 - - - _coordinate - (8, 75) - - - _rotation - 0 - - - - ha5kfu_execproc_xx - - id - ha5kfu_execproc_xx_0 - - - _enabled - True - - - type - cc - - - commandline - "csdr shift_math_cc %g"%rate - - - alias - - - - affinity - - - - minoutbuf - 0 - - - maxoutbuf - 0 - - - _coordinate - (488, 251) - - - _rotation - 0 - - - - ha5kfu_execproc_xx - - id - ha5kfu_execproc_xx_0_0 - - - _enabled - True - - - type - cc - - - commandline - "csdr shift_addition_cc %g"%rate - - - alias - - - - affinity - - - - minoutbuf - 0 - - - maxoutbuf - 0 - - - _coordinate - (472, 411) - - - _rotation - 0 - - - - notebook - - id - nb0 - - - _enabled - True - - - style - wx.NB_TOP - - - labels - ['shift_math_cc', 'shift_addition_cc', 'shift_table_cc', 'decimating_shift_addition_cc','original'] - - - grid_pos - - - - notebook - - - - alias - - - - _coordinate - (272, 11) - - - _rotation - 0 - - - - wxgui_fftsink2 - - id - wxgui_fftsink2_0_0 - - - _enabled - True - - - type - complex - - - title - FFT Plot - - - samp_rate - samp_rate - - - baseband_freq - 0 - - - y_per_div - 10 - - - y_divs - 10 - - - ref_level - 0 - - - ref_scale - 2.0 - - - fft_size - 1024 - - - fft_rate - 15 - - - peak_hold - False - - - average - False - - - avg_alpha - 0 - - - win - None - - - win_size - - - - grid_pos - - - - notebook - nb0,4 - - - freqvar - None - - - alias - - - - affinity - - - - _coordinate - (496, 27) - - - _rotation - 0 + analog.GR_SIN_WAVE blocks_throttle - id - blocks_throttle_0 + alias + + + + comment + + + + affinity + _enabled True - type - complex + _coordinate + (224, 99) - samples_per_second - samp_rate + _rotation + 0 - vlen - 1 + id + blocks_throttle_0 ignoretag True - alias - - - - affinity - + maxoutbuf + 0 minoutbuf 0 - maxoutbuf - 0 - - - _coordinate - (224, 107) - - - _rotation - 0 - - - - wxgui_scopesink2 - - id - wxgui_scopesink2_0_0_0 - - - _enabled - True - - - type - complex - - - title - Scope Plot - - - samp_rate + samples_per_second samp_rate - - v_scale - 0 - - - v_offset - 0 - - - t_scale - 0 - - - ac_couple - False - - - xy_mode - False - - - num_inputs - 1 - - - win_size - - - - grid_pos - - - - notebook - nb0,2 - - - trig_mode - wxgui.TRIG_MODE_NORM - - - y_axis_label - Counts - - - alias - - - - affinity - - - - _coordinate - (1080, 899) - - - _rotation - 0 - - - - wxgui_fftsink2 - - id - wxgui_fftsink2_0_1_0_0 - - - _enabled - True - type complex - title - FFT Plot - - - samp_rate - samp_rate/decimation - - - baseband_freq - 0 - - - y_per_div - 10 - - - y_divs - 10 - - - ref_level - 0 - - - ref_scale - 2.0 - - - fft_size - 1024 - - - fft_rate - 15 - - - peak_hold - False - - - average - False - - - avg_alpha - 0 - - - win - None - - - win_size - - - - grid_pos - - - - notebook - nb0,3 - - - freqvar - None - - - alias - - - - affinity - - - - _coordinate - (320, 667) - - - _rotation - 0 - - - - wxgui_scopesink2 - - id - wxgui_scopesink2_0_0_0_0 - - - _enabled - True - - - type - complex - - - title - Scope Plot - - - samp_rate - samp_rate/decimation - - - v_scale - 0 - - - v_offset - 0 - - - t_scale - 0 - - - ac_couple - False - - - xy_mode - False - - - num_inputs + vlen 1 - - win_size - - - - grid_pos - - - - notebook - nb0,3 - - - trig_mode - wxgui.TRIG_MODE_NORM - - - y_axis_label - Counts - - - alias - - - - affinity - - - - _coordinate - (320, 883) - - - _rotation - 0 - ha5kfu_execproc_xx - id - ha5kfu_execproc_xx_0_0_0_0 + alias + + + + commandline + "csdr shift_math_cc %g"%rate + + + comment + + + + affinity + _enabled True + + _coordinate + (832, 99) + + + _rotation + 0 + + + id + ha5kfu_execproc_xx_0 + + + maxoutbuf + 0 + + + minoutbuf + 0 + type cc + + + ha5kfu_execproc_xx + + alias + + + + commandline + "csdr shift_addition_cc %g"%rate + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (824, 443) + + + _rotation + 0 + + + id + ha5kfu_execproc_xx_0_0 + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + type + cc + + + + ha5kfu_execproc_xx + + alias + + + + commandline + "csdr shift_table_cc %g 32768"%rate + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (824, 931) + + + _rotation + 0 + + + id + ha5kfu_execproc_xx_0_0_0 + + + maxoutbuf + 0 + + + minoutbuf + 0 + + + type + cc + + + + ha5kfu_execproc_xx + + alias + + commandline "csdr decimating_shift_addition_cc %g %d"%(rate, decimation) - alias + comment @@ -772,12 +487,8 @@ - minoutbuf - 0 - - - maxoutbuf - 0 + _enabled + True _coordinate @@ -787,83 +498,221 @@ _rotation 0 - - - ha5kfu_execproc_xx id - ha5kfu_execproc_xx_0_0_0 + ha5kfu_execproc_xx_0_0_0_0 - _enabled - True + maxoutbuf + 0 + + + minoutbuf + 0 type cc - - commandline - "csdr shift_table_cc %g 32768"%rate - + + + ha5kfu_execproc_xx alias + + commandline + "csdr shift_addfast_cc %g"%rate + + + comment + + affinity - minoutbuf + _enabled + True + + + _coordinate + (608, 603) + + + _rotation 0 + + id + ha5kfu_execproc_xx_0_1 + maxoutbuf 0 - _coordinate - (824, 771) + minoutbuf + 0 - _rotation - 0 + type + cc - wxgui_fftsink2 + ha5kfu_execproc_xx - id - wxgui_fftsink2_0_1_0 + alias + + + + commandline + "csdr shift_unroll_cc %g"%rate + + + comment + + + + affinity + _enabled True + + _coordinate + (104, 563) + + + _rotation + 0 + + + id + ha5kfu_execproc_xx_0_1_0 + + + maxoutbuf + 0 + + + minoutbuf + 0 + type - complex + cc + + + + notebook + + alias + - title - FFT Plot + comment + - samp_rate - samp_rate + _enabled + True + + + _coordinate + (272, 11) + + + _rotation + 0 + + + grid_pos + + + + id + nb0 + + + labels + ['shift_math_cc', 'shift_addition_cc', 'shift_table_cc', 'decimating_shift_addition_cc','shift_addfast_cc','shift_unroll_cc','original'] + + + notebook + + + + style + wx.NB_TOP + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False baseband_freq 0 - y_per_div - 10 + alias + - y_divs - 10 + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (1096, 19) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0 + + + notebook + nb0,0 + + + peak_hold + False ref_level @@ -873,54 +722,283 @@ ref_scale 2.0 - - fft_size - 1024 - fft_rate 15 - peak_hold - False + samp_rate + samp_rate - average - False + title + FFT Plot - avg_alpha - 0 - - - win - None + type + complex win_size - grid_pos - - - - notebook - nb0,2 - - - freqvar + win None + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + alias + + comment + + affinity + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (848, 187) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0_0 + + + notebook + nb0,6 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT Plot + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (1096, 355) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0_1 + + + notebook + nb0,1 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT Plot + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + _coordinate (1080, 691) @@ -929,123 +1007,21 @@ _rotation 0 - - - wxgui_scopesink2 - - id - wxgui_scopesink2_0_0 - - - _enabled - True - - - type - complex - - - title - Scope Plot - - - samp_rate - samp_rate - - - v_scale - 0 - - - v_offset - 0 - - - t_scale - 0 - - - ac_couple - False - - - xy_mode - False - - - num_inputs - 1 - - - win_size - - grid_pos - - notebook - nb0,1 - - - trig_mode - wxgui.TRIG_MODE_NORM - - - y_axis_label - Counts - - - alias - - - - affinity - - - - _coordinate - (824, 571) - - - _rotation - 0 - - - - wxgui_fftsink2 id - wxgui_fftsink2_0_1 + wxgui_fftsink2_0_1_0 - _enabled - True + notebook + nb0,2 - type - complex - - - title - FFT Plot - - - samp_rate - samp_rate - - - baseband_freq - 0 - - - y_per_div - 10 - - - y_divs - 10 + peak_hold + False ref_level @@ -1055,244 +1031,869 @@ ref_scale 2.0 - - fft_size - 1024 - fft_rate 15 - peak_hold - False + samp_rate + samp_rate - average - False + title + FFT Plot - avg_alpha - 0 - - - win - None + type + complex win_size - grid_pos - - - - notebook - nb0,1 - - - freqvar + win None + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + alias + + comment + + affinity + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + _coordinate - (824, 355) + (320, 667) _rotation 0 + + grid_pos + + + + id + wxgui_fftsink2_0_1_0_0 + + + notebook + nb0,3 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate/decimation + + + title + FFT Plot + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (840, 523) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0_1_1 + + + notebook + nb0,4 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT Plot + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + + + + wxgui_fftsink2 + + avg_alpha + 0 + + + average + False + + + baseband_freq + 0 + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + fft_size + 1024 + + + freqvar + None + + + _coordinate + (336, 315) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_fftsink2_0_1_1_0 + + + notebook + nb0,5 + + + peak_hold + False + + + ref_level + 0 + + + ref_scale + 2.0 + + + fft_rate + 15 + + + samp_rate + samp_rate + + + title + FFT Plot + + + type + complex + + + win_size + + + + win + None + + + y_divs + 10 + + + y_per_div + 10 + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (1096, 235) + + + _rotation + 0 + + + grid_pos + + id wxgui_scopesink2_0 - _enabled - True - - - type - complex - - - title - Scope Plot - - - samp_rate - samp_rate - - - v_scale - 0 - - - v_offset - 0 - - - t_scale - 0 - - - ac_couple - False - - - xy_mode - False + notebook + nb0,0 num_inputs 1 - win_size - + samp_rate + samp_rate - grid_pos - + t_scale + 0 - notebook - nb0,0 + title + Scope Plot trig_mode wxgui.TRIG_MODE_NORM - - y_axis_label - Counts - - - alias - - - - affinity - - - - _coordinate - (824, 235) - - - _rotation - 0 - - - - wxgui_fftsink2 - - id - wxgui_fftsink2_0 - - - _enabled - True - type complex - title - FFT Plot - - - samp_rate - samp_rate - - - baseband_freq + v_offset 0 - y_per_div - 10 - - - y_divs - 10 - - - ref_level + v_scale 0 - - ref_scale - 2.0 - - - fft_size - 1024 - - - fft_rate - 15 - - - peak_hold - False - - - average - False - - - avg_alpha - 0 - - - win - None - win_size - grid_pos - + xy_mode + False - notebook - nb0,0 + y_axis_label + Counts + + + wxgui_scopesink2 - freqvar - None + ac_couple + False alias + + comment + + affinity + + _enabled + True + _coordinate - (824, 19) + (1096, 571) _rotation 0 + + grid_pos + + + + id + wxgui_scopesink2_0_0 + + + notebook + nb0,1 + + + num_inputs + 1 + + + samp_rate + samp_rate + + + t_scale + 0 + + + title + Scope Plot + + + trig_mode + wxgui.TRIG_MODE_NORM + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + + + + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (1080, 899) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_scopesink2_0_0_0 + + + notebook + nb0,2 + + + num_inputs + 1 + + + samp_rate + samp_rate + + + t_scale + 0 + + + title + Scope Plot + + + trig_mode + wxgui.TRIG_MODE_NORM + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + + + + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (320, 883) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_scopesink2_0_0_0_0 + + + notebook + nb0,3 + + + num_inputs + 1 + + + samp_rate + samp_rate/decimation + + + t_scale + 0 + + + title + Scope Plot + + + trig_mode + wxgui.TRIG_MODE_NORM + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + + + + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (840, 747) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_scopesink2_0_0_1 + + + notebook + nb0,4 + + + num_inputs + 1 + + + samp_rate + samp_rate + + + t_scale + 0 + + + title + Scope Plot + + + trig_mode + wxgui.TRIG_MODE_NORM + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + + + + wxgui_scopesink2 + + ac_couple + False + + + alias + + + + comment + + + + affinity + + + + _enabled + True + + + _coordinate + (336, 531) + + + _rotation + 0 + + + grid_pos + + + + id + wxgui_scopesink2_0_0_1_0 + + + notebook + nb0,5 + + + num_inputs + 1 + + + samp_rate + samp_rate + + + t_scale + 0 + + + title + Scope Plot + + + trig_mode + wxgui.TRIG_MODE_NORM + + + type + complex + + + v_offset + 0 + + + v_scale + 0 + + + win_size + + + + xy_mode + False + + + y_axis_label + Counts + analog_sig_source_x_0 @@ -1300,48 +1901,18 @@ 0 0 - - blocks_throttle_0 - wxgui_fftsink2_0_0 - 0 - 0 - - - ha5kfu_execproc_xx_0 - wxgui_scopesink2_0 - 0 - 0 - - - ha5kfu_execproc_xx_0 - wxgui_fftsink2_0 - 0 - 0 - blocks_throttle_0 ha5kfu_execproc_xx_0 0 0 - - ha5kfu_execproc_xx_0_0 - wxgui_fftsink2_0_1 - 0 - 0 - blocks_throttle_0 ha5kfu_execproc_xx_0_0 0 0 - - ha5kfu_execproc_xx_0_0 - wxgui_scopesink2_0_0 - 0 - 0 - blocks_throttle_0 ha5kfu_execproc_xx_0_0_0 @@ -1355,14 +1926,44 @@ 0 - ha5kfu_execproc_xx_0_0_0_0 - wxgui_scopesink2_0_0_0_0 + blocks_throttle_0 + ha5kfu_execproc_xx_0_1 0 0 - ha5kfu_execproc_xx_0_0_0_0 - wxgui_fftsink2_0_1_0_0 + blocks_throttle_0 + ha5kfu_execproc_xx_0_1_0 + 0 + 0 + + + blocks_throttle_0 + wxgui_fftsink2_0_0 + 0 + 0 + + + ha5kfu_execproc_xx_0 + wxgui_fftsink2_0 + 0 + 0 + + + ha5kfu_execproc_xx_0 + wxgui_scopesink2_0 + 0 + 0 + + + ha5kfu_execproc_xx_0_0 + wxgui_fftsink2_0_1 + 0 + 0 + + + ha5kfu_execproc_xx_0_0 + wxgui_scopesink2_0_0 0 0 @@ -1378,4 +1979,40 @@ 0 0 + + ha5kfu_execproc_xx_0_0_0_0 + wxgui_fftsink2_0_1_0_0 + 0 + 0 + + + ha5kfu_execproc_xx_0_0_0_0 + wxgui_scopesink2_0_0_0_0 + 0 + 0 + + + ha5kfu_execproc_xx_0_1 + wxgui_fftsink2_0_1_1 + 0 + 0 + + + ha5kfu_execproc_xx_0_1 + wxgui_scopesink2_0_0_1 + 0 + 0 + + + ha5kfu_execproc_xx_0_1_0 + wxgui_fftsink2_0_1_1_0 + 0 + 0 + + + ha5kfu_execproc_xx_0_1_0 + wxgui_scopesink2_0_0_1_0 + 0 + 0 + diff --git a/libcsdr.c b/libcsdr.c index df87266..150a7e6 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -394,6 +394,44 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ #else + +#if 1 + +#define SADF_L1(j) cos_vals_ ## j = cos_start * dcos_ ## j - sin_start * dsin_ ## j; \ + sin_vals_ ## j = sin_start * dcos_ ## j + cos_start * dsin_ ## j; +#define SADF_L2(j) iof(output,4*i+j)=(cos_vals_ ## j)*iof(input,4*i+j)-(sin_vals_ ## j)*qof(input,4*i+j); \ + qof(output,4*i+j)=(sin_vals_ ## j)*iof(input,4*i+j)+(cos_vals_ ## j)*qof(input,4*i+j); + +float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_addfast_data_t* d, float starting_phase) +{ + //input_size should be multiple of 4 + //fprintf(stderr, "shift_addfast_cc: input_size = %d\n", input_size); + float cos_start=cos(starting_phase); + float sin_start=sin(starting_phase); + float register cos_vals_0, cos_vals_1, cos_vals_2, cos_vals_3, + sin_vals_0, sin_vals_1, sin_vals_2, sin_vals_3, + dsin_0 = d->dsin[0], dsin_1 = d->dsin[1], dsin_2 = d->dsin[2], dsin_3 = d->dsin[3], + dcos_0 = d->dcos[0], dcos_1 = d->dcos[1], dcos_2 = d->dcos[2], dcos_3 = d->dcos[3]; + + for(int i=0;iphase_increment; + while(starting_phase>PI) starting_phase-=2*PI; + while(starting_phase<-PI) starting_phase+=2*PI; + return starting_phase; +} +#else float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_addfast_data_t* d, float starting_phase) { //input_size should be multiple of 4 @@ -421,6 +459,7 @@ float shift_addfast_cc(complexf *input, complexf* output, int input_size, shift_ while(starting_phase<-PI) starting_phase+=2*PI; return starting_phase; } +#endif #endif diff --git a/test200.c b/test200.c index 9feb457..d2da851 100644 --- a/test200.c +++ b/test200.c @@ -70,6 +70,15 @@ int main() clock_gettime(CLOCK_MONOTONIC_RAW, &end_time); fprintf(stderr,"shift_math_cc done in %g seconds.\n",TIME_TAKEN(start_time,end_time)); + //shift_table_cc + shift_table_data_t shift_table_data=shift_table_init(65536); + starting_phase = 0; + + clock_gettime(CLOCK_MONOTONIC_RAW, &start_time); + for(int i=0;i Date: Wed, 2 Dec 2015 14:51:20 +0100 Subject: [PATCH 23/74] Working hard on ddcd. --- Makefile | 1 + ddcd.cpp | 144 +++++++++++++++++++++++++++++++++++++++++++++---------- ddcd.h | 21 ++++++++ 3 files changed, 142 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 64f52ac..09cd72b 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,7 @@ install: all install -m 0755 libcsdr.so /usr/lib install -m 0755 csdr /usr/bin install -m 0755 csdr-fm /usr/bin + install -m 0755 ddcd /usr/bin ldconfig uninstall: rm /usr/lib/libcsdr.so /usr/bin/csdr /usr/bin/csdr-fm diff --git a/ddcd.cpp b/ddcd.cpp index 37242e5..2bf89c1 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -37,9 +37,16 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. int host_port = 0; char host_address[100] = "127.0.0.1"; int decimation = 0; +float transition_bw = 0.05; int bufsize = 1024; int bufsizeall; int pipe_max_size; +char ddc_method_str[100] = "td"; +ddc_method_t ddc_method; +pid_t main_dsp_proc; +pid_t pgrp; +int input_pipe = STDIN_FILENO; //can be stdin, or the stdout of main_subprocess + char* buf; int set_nonblocking(int fd) @@ -71,7 +78,9 @@ int main(int argc, char* argv[]) {"port", required_argument, 0, 'p' }, {"address", required_argument, 0, 'a' }, {"decimation", required_argument, 0, 'd' }, - {"bufsize", required_argument, 0, 'b' } + {"bufsize", required_argument, 0, 'b' }, + {"method", required_argument, 0, 'm' } + {"transition", required_argument, 0, 't' } }; c = getopt_long(argc, argv, "p:a:d:b:", long_options, &option_index); if(c==-1) break; @@ -90,17 +99,38 @@ int main(int argc, char* argv[]) case 'b': bufsize=atoi(optarg); break; + case 'm': + host_address[100-1]=0; + strncpy(ddc_method_str,optarg,100-1); + break; + case 't': + sscanf(optarg,"%g",&transition_bw); + break; case 0: case '?': case ':': - default: - printf(" 0%o ??\n", c); + default:; } } - if(!decimation) error_exit(MSG_START "missing required command line argument, --decimation.\n"); - if(!host_port) error_exit(MSG_START "missing required command line argument, --port.\n"); + if(!decimation) print_exit(MSG_START "missing required command line argument, --decimation.\n"); + if(!host_port) print_exit(MSG_START "missing required command line argument, --port.\n"); + if(decimation<0) print_exit(MSG_START "invalid value for --decimation (should be >0).\n"); + if(decimation==1) fprintf(stderr, MSG_START "decimation = 1, just copying raw samples.\n"); + if(transition_bw<0||transition_bw>0.5) print_exit(MSG_START "invalid value for --transition (should be between 0 and 0.5).\n"); + if(!strcmp(ddc_method_str,"td")) + { + ddc_method = M_TD; + fprintf(stderr, MSG_START "method is M_TD (default).\n"); + } + else if (!strcmp(ddc_method_str,"fastddc")) + { + ddc_method = M_FASTDDC; + fprintf(stderr, MSG_START "method is M_FASTDDC.\n"); + } + else print_exit(MSG_START "invalid parameter given to --method.\n"); + struct sockaddr_in addr_host; int listen_socket; std::vector clients; @@ -109,7 +139,7 @@ int main(int argc, char* argv[]) int sockopt = 1; if( setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&sockopt, sizeof(sockopt)) == -1 ) - error_exit(MSG_START "cannot set SO_REUSEADDR.\n"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 + error_exit(MSG_START "cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 memset(&addr_host,'0',sizeof(addr_host)); addr_host.sin_family=AF_INET; @@ -117,13 +147,13 @@ int main(int argc, char* argv[]) addr_host.sin_addr.s_addr = INADDR_ANY; if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) - error_exit(MSG_START "invalid host address.\n"); + error_exit(MSG_START "invalid host address"); if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) - error_exit(MSG_START "cannot bind() address to the socket.\n"); + error_exit(MSG_START "cannot bind() address to the socket"); if( listen(listen_socket, 10) == -1 ) - error_exit(MSG_START "cannot listen() on socket.\n"); + error_exit(MSG_START "cannot listen() on socket"); fprintf(stderr,MSG_START "listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); @@ -131,18 +161,9 @@ int main(int argc, char* argv[]) socklen_t addr_cli_len = sizeof(addr_cli); int new_socket; - //Set stdin and listen_socket to non-blocking - if(set_nonblocking(STDIN_FILENO) || set_nonblocking(listen_socket)) - error_exit(MSG_START "cannot set_nonblocking().\n"); - bufsizeall = bufsize*sizeof(char); buf = (char*)malloc(bufsizeall); - FD_ZERO(&select_fds); - FD_SET(listen_socket, &select_fds); - FD_SET(STDIN_FILENO, &select_fds); - int highfd = ((listen_socket>STDIN_FILENO)?listen_socket:STDIN_FILENO) + 1; - FILE* tempfile = fopen("/proc/sys/fs/pipe-max-size","r"); if(!tempfile) { @@ -159,6 +180,46 @@ int main(int argc, char* argv[]) // perror("failed to fcntl(STDIN_FILENO, F_SETPIPE_SZ, ...)"); } + //We'll see if it is a good idea: + setpgrp(); + pgrp = getpgrp(); + + //Start DSP subprocess from the main process if required + char main_subprocess_cmd_buf[500]; + pid_t main_subprocess_pid = 0; + + int pipe_m2s_ctl[2]; //main to subprocess :: control channel + int pipe_s2m[2]; //subprocess to main + + if(pipe(pipe_m2s_ctl)) error_exit(MSG_START "couldn't create pipe_m2s_ctl"); + if(pipe(pipe_s2m)) error_exit(MSG_START "couldn't create pipe_s2m"); + + if(decimation!=1) + { + switch(ddc_method) + { + case M_TD: + break; + case M_FASTDDC: + sprintf(main_subprocess_cmd_buf, subprocess_args_fastddc_1, decimation, tansition_bw); + close(STDIN_FILENO); // redirect stdin to the stdin of the subprocess + main_subprocess_pid = run_subprocess( main_subprocess_cmd_buf, 0, pipe_s2m ); + break; + } + } + + int highfd = 0; + FD_ZERO(&select_fds); + FD_SET(listen_socket, &select_fds); + maxfd(&highfd, listen_socket); + if(main_subprocess_pid) input_pipe = pipe_s2m[1]; //else STDIN_FILENO + FD_SET(input_pipe, &select_fds); + maxfd(&highfd, input_pipe); + + //Set stdin and listen_socket to non-blocking + if(set_nonblocking(input_pipe) || set_nonblocking(listen_socket)) //don't do it before subprocess fork! + error_exit(MSG_START "cannot set_nonblocking()"); + for(;;) { //Let's wait until there is any new data to read, or any new connection! @@ -173,17 +234,17 @@ int main(int argc, char* argv[]) this_client->socket = new_socket; if(pipe(this_client->pipefd) == -1) { - perror(MSG_START "cannot open new pipe() for the client.\n"); + perror(MSG_START "cannot open new pipe() for the client"); continue; } if(fcntl(this_client->pipefd[1], F_SETPIPE_SZ, pipe_max_size) == -1) - perror("failed to F_SETPIPE_SZ for the client pipe!"); + perror("failed to F_SETPIPE_SZ for the client pipe"); if(this_client->pid = fork()) { //We're the parent set_nonblocking(this_client->pipefd[1]); clients.push_back(this_client); - printf("client pid: %d\n", this_client->pid); + fprintf(stderr, MSG_START "client pid: %d\n", this_client->pid); } else { @@ -193,7 +254,7 @@ int main(int argc, char* argv[]) } } - int retval = read(STDIN_FILENO, buf, bufsizeall); + int retval = read(input_pipe, buf, bufsizeall); if(retval==0) { //end of input stream, close clients and exit @@ -236,7 +297,31 @@ int main(int argc, char* argv[]) // } - return 0; + return 0; +} + +pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out) +{ + pid_t pid = fork(); + if(p < 0) return 0; //fork failed + if(p==0) + { + //We're the subprocess + if(fcntl(pipe_in[1], F_SETPIPE_SZ, pipe_max_size) == -1) perror("Failed to F_SETPIPE_SZ in run_subprocess()"); + if(pipe_in) + { + close(pipe_in[1]); + dup2(pipe_in[0], STDIN_FILENO); + } + if(pipe_out) + { + close(pipe_out[0]); + dup2(pipe_out[1], STDOUT_FILENO); + } + execl("/bin/bash","bash","-c",cmd, 0); + error_exit(MSG_START "run_subprocess failed to execute command") + } + else return pid; } void print_client(client_t* client, const char* what) @@ -251,7 +336,7 @@ void client_cleanup() void client() { - fprintf(stderr, "I'm the client\n"); + print_client(this_client, "client process forked."); for(;;) { read(this_client->pipefd[0],buf,bufsizeall); @@ -268,3 +353,14 @@ void error_exit(const char* why) perror(why); exit(1); } + +void print_exit(const char* why) +{ + fprintf(stderr, "%s", why); + exit(1); +} + +void maxfd(int* maxfd, int fd) +{ + if(fd>*maxfd) *maxfd=fd+1; +} diff --git a/ddcd.h b/ddcd.h index 0767500..eea134d 100644 --- a/ddcd.h +++ b/ddcd.h @@ -23,10 +23,31 @@ typedef struct client_s pid_t pid; int pipefd[2]; int error; + pid_t dsp_proc; } client_t; void client(); void error_exit(const char* why); +void print_exit(const char* why); void print_client(client_t* client, const char* what); int proc_exists(pid_t pid); +void run_subprocess(char* cmd, int* pipe_in, int* pipe_out); +void maxfd(int* maxfd, int fd); + +typedef enum ddc_method_e +{ + M_TD, + M_FASTDDC +} ddc_method_t; + +const char subprocess_cmd_td[] = "csdr " +#ifdef NEON_OPTS + "shift_addfast_cc" +#else + "shift_unroll_cc" +#endif + " --pipe %d,%d | csdr fir_decimate_cc %d %g" }; + +const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; +const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc %d --pipe %d,%d %g"; From c6e0a8f5c833dce8be7af45d28f0fb41d8819d73 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 2 Dec 2015 15:27:28 +0100 Subject: [PATCH 24/74] Fixed build. --- ddcd.cpp | 12 ++++++------ ddcd.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index 2bf89c1..968941f 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -79,7 +79,7 @@ int main(int argc, char* argv[]) {"address", required_argument, 0, 'a' }, {"decimation", required_argument, 0, 'd' }, {"bufsize", required_argument, 0, 'b' }, - {"method", required_argument, 0, 'm' } + {"method", required_argument, 0, 'm' }, {"transition", required_argument, 0, 't' } }; c = getopt_long(argc, argv, "p:a:d:b:", long_options, &option_index); @@ -201,7 +201,7 @@ int main(int argc, char* argv[]) case M_TD: break; case M_FASTDDC: - sprintf(main_subprocess_cmd_buf, subprocess_args_fastddc_1, decimation, tansition_bw); + sprintf(main_subprocess_cmd_buf, subprocess_args_fastddc_1, decimation, transition_bw); close(STDIN_FILENO); // redirect stdin to the stdin of the subprocess main_subprocess_pid = run_subprocess( main_subprocess_cmd_buf, 0, pipe_s2m ); break; @@ -303,8 +303,8 @@ int main(int argc, char* argv[]) pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out) { pid_t pid = fork(); - if(p < 0) return 0; //fork failed - if(p==0) + if(pid < 0) return 0; //fork failed + if(pid == 0) { //We're the subprocess if(fcntl(pipe_in[1], F_SETPIPE_SZ, pipe_max_size) == -1) perror("Failed to F_SETPIPE_SZ in run_subprocess()"); @@ -318,8 +318,8 @@ pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out) close(pipe_out[0]); dup2(pipe_out[1], STDOUT_FILENO); } - execl("/bin/bash","bash","-c",cmd, 0); - error_exit(MSG_START "run_subprocess failed to execute command") + execl("/bin/bash","bash","-c",cmd, NULL); + error_exit(MSG_START "run_subprocess failed to execute command"); } else return pid; } diff --git a/ddcd.h b/ddcd.h index eea134d..5bdd085 100644 --- a/ddcd.h +++ b/ddcd.h @@ -32,7 +32,7 @@ void error_exit(const char* why); void print_exit(const char* why); void print_client(client_t* client, const char* what); int proc_exists(pid_t pid); -void run_subprocess(char* cmd, int* pipe_in, int* pipe_out); +pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out); void maxfd(int* maxfd, int fd); typedef enum ddc_method_e @@ -47,7 +47,7 @@ const char subprocess_cmd_td[] = "csdr " #else "shift_unroll_cc" #endif - " --pipe %d,%d | csdr fir_decimate_cc %d %g" }; + " --pipe %d,%d | csdr fir_decimate_cc %d %g"; const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc %d --pipe %d,%d %g"; From 2a6b1c2073379e2b7581afd40c9608b8dffcdf63 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 2 Dec 2015 19:38:17 +0100 Subject: [PATCH 25/74] Handle signals from ddcd. --- ddcd.cpp | 19 +++++++++++++++++++ ddcd.h | 5 ++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/ddcd.cpp b/ddcd.cpp index 968941f..4a9ad88 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -64,6 +64,15 @@ int proc_exists(pid_t pid) return kill(pid, 0) != -1; } +void sig_handler(int signo) +{ + if(pgrp!=1 && pgrp!=0) //I just want to make sure that we cannot kill init or sched + killpg(pgrp, signo); + fprintf(stderr, MSG_START "signal caught, exiting ddcd...\n"); + fflush(stderr); + exit(0); +} + client_t* this_client; int main(int argc, char* argv[]) @@ -131,6 +140,15 @@ int main(int argc, char* argv[]) } else print_exit(MSG_START "invalid parameter given to --method.\n"); + //set signals + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sig_handler; + sigaction(SIGKILL, &sa, NULL); + sigaction(SIGQUIT, &sa, NULL); + sigaction(SIGHUP, &sa, NULL); + prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits + struct sockaddr_in addr_host; int listen_socket; std::vector clients; @@ -202,6 +220,7 @@ int main(int argc, char* argv[]) break; case M_FASTDDC: sprintf(main_subprocess_cmd_buf, subprocess_args_fastddc_1, decimation, transition_bw); + fprintf(stderr, MSG_START "starting main_subprocess_cmd: %s\n", main_subprocess_cmd_buf); close(STDIN_FILENO); // redirect stdin to the stdin of the subprocess main_subprocess_pid = run_subprocess( main_subprocess_cmd_buf, 0, pipe_s2m ); break; diff --git a/ddcd.h b/ddcd.h index 5bdd085..4d6bb33 100644 --- a/ddcd.h +++ b/ddcd.h @@ -15,6 +15,7 @@ #include #include #include +#include typedef struct client_s { @@ -34,6 +35,7 @@ void print_client(client_t* client, const char* what); int proc_exists(pid_t pid); pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out); void maxfd(int* maxfd, int fd); +void sig_handler(int signo); typedef enum ddc_method_e { @@ -49,5 +51,6 @@ const char subprocess_cmd_td[] = "csdr " #endif " --pipe %d,%d | csdr fir_decimate_cc %d %g"; -const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; +const char subprocess_args_fastddc_1[] = "csdr through %d %g"; +//const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc %d --pipe %d,%d %g"; From 558cbbfefec4917cfb03fa6ee7f73a939206c9ed Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 2 Dec 2015 21:58:48 +0100 Subject: [PATCH 26/74] main subprocess now works. --- ddcd.cpp | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index 4a9ad88..c226503 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -46,6 +46,7 @@ ddc_method_t ddc_method; pid_t main_dsp_proc; pid_t pgrp; int input_pipe = STDIN_FILENO; //can be stdin, or the stdout of main_subprocess +pid_t main_subprocess_pid = 0; char* buf; @@ -66,6 +67,13 @@ int proc_exists(pid_t pid) void sig_handler(int signo) { + int tmpstat; + if(signo==SIGCHLD) + if( main_subprocess_pid && signo==SIGCHLD && (waitpid(main_subprocess_pid, &tmpstat, WNOHANG), 1) && !proc_exists(main_subprocess_pid) ) + { + fprintf(stderr,MSG_START "main_subprocess_pid exited! Exiting...\n"); + } + else return; if(pgrp!=1 && pgrp!=0) //I just want to make sure that we cannot kill init or sched killpg(pgrp, signo); fprintf(stderr, MSG_START "signal caught, exiting ddcd...\n"); @@ -91,7 +99,7 @@ int main(int argc, char* argv[]) {"method", required_argument, 0, 'm' }, {"transition", required_argument, 0, 't' } }; - c = getopt_long(argc, argv, "p:a:d:b:", long_options, &option_index); + c = getopt_long(argc, argv, "p:a:d:b:m:t:", long_options, &option_index); if(c==-1) break; switch (c) { @@ -109,7 +117,7 @@ int main(int argc, char* argv[]) bufsize=atoi(optarg); break; case 'm': - host_address[100-1]=0; + ddc_method_str[100-1]=0; strncpy(ddc_method_str,optarg,100-1); break; case 't': @@ -119,6 +127,7 @@ int main(int argc, char* argv[]) case '?': case ':': default:; + print_exit(MSG_START "error in getopt_long()\n"); } } @@ -128,7 +137,8 @@ int main(int argc, char* argv[]) if(decimation==1) fprintf(stderr, MSG_START "decimation = 1, just copying raw samples.\n"); if(transition_bw<0||transition_bw>0.5) print_exit(MSG_START "invalid value for --transition (should be between 0 and 0.5).\n"); - if(!strcmp(ddc_method_str,"td")) + if(decimation==1); //don't do anything then + else if(!strcmp(ddc_method_str,"td")) { ddc_method = M_TD; fprintf(stderr, MSG_START "method is M_TD (default).\n"); @@ -144,9 +154,13 @@ int main(int argc, char* argv[]) struct sigaction sa; memset(&sa, 0, sizeof(sa)); sa.sa_handler = sig_handler; + sigaction(SIGTERM, &sa, NULL); sigaction(SIGKILL, &sa, NULL); sigaction(SIGQUIT, &sa, NULL); + sigaction(SIGINT, &sa, NULL); sigaction(SIGHUP, &sa, NULL); + sigaction(SIGCHLD, &sa, NULL); + //sigaction(SIGPIPE, &sa, NULL); prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits struct sockaddr_in addr_host; @@ -204,7 +218,7 @@ int main(int argc, char* argv[]) //Start DSP subprocess from the main process if required char main_subprocess_cmd_buf[500]; - pid_t main_subprocess_pid = 0; + int pipe_m2s_ctl[2]; //main to subprocess :: control channel int pipe_s2m[2]; //subprocess to main @@ -221,8 +235,9 @@ int main(int argc, char* argv[]) case M_FASTDDC: sprintf(main_subprocess_cmd_buf, subprocess_args_fastddc_1, decimation, transition_bw); fprintf(stderr, MSG_START "starting main_subprocess_cmd: %s\n", main_subprocess_cmd_buf); + if(!(main_subprocess_pid = run_subprocess( main_subprocess_cmd_buf, 0, pipe_s2m ))) + print_exit(MSG_START "couldn't start main_subprocess_cmd!\n"); close(STDIN_FILENO); // redirect stdin to the stdin of the subprocess - main_subprocess_pid = run_subprocess( main_subprocess_cmd_buf, 0, pipe_s2m ); break; } } @@ -231,7 +246,7 @@ int main(int argc, char* argv[]) FD_ZERO(&select_fds); FD_SET(listen_socket, &select_fds); maxfd(&highfd, listen_socket); - if(main_subprocess_pid) input_pipe = pipe_s2m[1]; //else STDIN_FILENO + if(main_subprocess_pid) input_pipe = pipe_s2m[0]; //else STDIN_FILENO FD_SET(input_pipe, &select_fds); maxfd(&highfd, input_pipe); @@ -313,7 +328,6 @@ int main(int argc, char* argv[]) } } //TODO: at the end, server closes pipefd[1] for client - // } return 0; @@ -322,11 +336,13 @@ int main(int argc, char* argv[]) pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out) { pid_t pid = fork(); + //fprintf(stderr, "run_subprocess :: fork-ed %d\n", pid); if(pid < 0) return 0; //fork failed if(pid == 0) { //We're the subprocess - if(fcntl(pipe_in[1], F_SETPIPE_SZ, pipe_max_size) == -1) perror("Failed to F_SETPIPE_SZ in run_subprocess()"); + //fprintf(stderr, "run_subprocess :: execl\n"); + //if(fcntl(pipe_in[1], F_SETPIPE_SZ, pipe_max_size) == -1) perror("Failed to F_SETPIPE_SZ in run_subprocess()"); if(pipe_in) { close(pipe_in[1]); @@ -337,7 +353,7 @@ pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out) close(pipe_out[0]); dup2(pipe_out[1], STDOUT_FILENO); } - execl("/bin/bash","bash","-c",cmd, NULL); + execl("/bin/bash","bash","-c",cmd, (char*)0); error_exit(MSG_START "run_subprocess failed to execute command"); } else return pid; @@ -381,5 +397,5 @@ void print_exit(const char* why) void maxfd(int* maxfd, int fd) { - if(fd>*maxfd) *maxfd=fd+1; + if(fd>=*maxfd) *maxfd=fd+1; } From c9406d38c91a273351c087e6fdb2ae1ece68071d Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 2 Dec 2015 22:40:20 +0100 Subject: [PATCH 27/74] Added client subprocess. --- ddcd.cpp | 48 ++++++++++++++++++++++++++++++++++++++++-------- ddcd.h | 5 +++-- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index c226503..61ddbbc 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -41,11 +41,12 @@ float transition_bw = 0.05; int bufsize = 1024; int bufsizeall; int pipe_max_size; +int in_client = 0; char ddc_method_str[100] = "td"; ddc_method_t ddc_method; pid_t main_dsp_proc; pid_t pgrp; -int input_pipe = STDIN_FILENO; //can be stdin, or the stdout of main_subprocess +int input_fd = STDIN_FILENO; //can be stdin, or the stdout of main_subprocess pid_t main_subprocess_pid = 0; char* buf; @@ -76,7 +77,7 @@ void sig_handler(int signo) else return; if(pgrp!=1 && pgrp!=0) //I just want to make sure that we cannot kill init or sched killpg(pgrp, signo); - fprintf(stderr, MSG_START "signal caught, exiting ddcd...\n"); + fprintf(stderr, MSG_START "signal %d caught in %s, exiting ddcd...\n", signo, (in_client)?"client":"main"); fflush(stderr); exit(0); } @@ -246,12 +247,12 @@ int main(int argc, char* argv[]) FD_ZERO(&select_fds); FD_SET(listen_socket, &select_fds); maxfd(&highfd, listen_socket); - if(main_subprocess_pid) input_pipe = pipe_s2m[0]; //else STDIN_FILENO - FD_SET(input_pipe, &select_fds); - maxfd(&highfd, input_pipe); + if(main_subprocess_pid) input_fd = pipe_s2m[0]; //else STDIN_FILENO + FD_SET(input_fd, &select_fds); + maxfd(&highfd, input_fd); //Set stdin and listen_socket to non-blocking - if(set_nonblocking(input_pipe) || set_nonblocking(listen_socket)) //don't do it before subprocess fork! + if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) //don't do it before subprocess fork! error_exit(MSG_START "cannot set_nonblocking()"); for(;;) @@ -288,7 +289,7 @@ int main(int argc, char* argv[]) } } - int retval = read(input_pipe, buf, bufsizeall); + int retval = read(input_fd, buf, bufsizeall); if(retval==0) { //end of input stream, close clients and exit @@ -371,13 +372,44 @@ void client_cleanup() void client() { + in_client=1; print_client(this_client, "client process forked."); + + char client_subprocess_cmd_buf[500]; + pid_t client_subprocess_pid = 0; + int input_fd = this_client->pipefd[0]; + + prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits + + if(decimation!=1) + { + int pipe_ctl[2], pipe_stdout[2]; + if(pipe(pipe_ctl)==-1) error_exit(MSG_START "cannot open new pipe() for the client subprocess"); + if(pipe(pipe_stdout)==-1) error_exit(MSG_START "cannot open new pipe() for the client subprocess"); + switch(ddc_method) + { + case M_TD: + sprintf(client_subprocess_cmd_buf, subprocess_cmd_td, pipe_ctl[0], decimation, transition_bw); + break; + case M_FASTDDC: + sprintf(client_subprocess_cmd_buf, subprocess_args_fastddc_2, decimation, pipe_ctl[0], transition_bw); + break; + } + if(!(client_subprocess_pid = run_subprocess( client_subprocess_cmd_buf, this_client->pipefd, pipe_stdout ))) + print_exit(MSG_START "couldn't start client_subprocess_cmd!\n"); + fprintf(stderr, MSG_START "starting client_subprocess_cmd: %s\n", client_subprocess_cmd_buf); + input_fd = pipe_stdout[0]; //we don't have to set it nonblocking + fprintf(stderr, MSG_START "pipe_stdout[0] = %d\n", pipe_stdout[0]); + setpgrp(); + pgrp = getpgrp(); + } for(;;) { - read(this_client->pipefd[0],buf,bufsizeall); + read(input_fd,buf,bufsizeall); if(send(this_client->socket,buf,bufsizeall,0)==-1) { print_client(this_client, "client process is exiting."); + if(client_subprocess_pid && pgrp!=1 && pgrp!=0) killpg(pgrp, SIGTERM); exit(0); } } diff --git a/ddcd.h b/ddcd.h index 4d6bb33..f67a4c6 100644 --- a/ddcd.h +++ b/ddcd.h @@ -49,8 +49,9 @@ const char subprocess_cmd_td[] = "csdr " #else "shift_unroll_cc" #endif - " --pipe %d,%d | csdr fir_decimate_cc %d %g"; + " --pipe %d | csdr fir_decimate_cc %d %g"; const char subprocess_args_fastddc_1[] = "csdr through %d %g"; //const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; -const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc %d --pipe %d,%d %g"; +//const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc %d --pipe %d %g"; +const char subprocess_args_fastddc_2[] = "csdr convert_u8_f %d %d %g"; From 4800710443e67cf208f675ee293d6f9da52d5f0d Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 2 Dec 2015 23:33:35 +0100 Subject: [PATCH 28/74] Fixed exit on Ctrl+C (SIGINT). --- csdr.c | 16 ++++++++++++++++ ddcd.cpp | 48 ++++++++++++++++++++++++++++++++++-------------- ddcd.h | 7 ++++--- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/csdr.c b/csdr.c index e74fadc..f6e4799 100644 --- a/csdr.c +++ b/csdr.c @@ -169,6 +169,22 @@ int init_fifo(int argc, char *argv[]) fcntl(fd, F_SETFL, flags | O_NONBLOCK); return fd; } + else if(!strcmp(argv[2],"--fd")) + { + //to use this: + //1. Create a pipe(pipedesc) in your process. + //2. fork() and execl() your process to run csdr, and give pipedesc[0] as parameter after --fd + // Note: when forking, the child process will get a copy of the file descriptor table! That's why this + // works at all, as file descriptor indexes are normally not transferable between processes, except for a *NIX socket way which is quite complicated... + //3. From your parent process, write into pipedesc[1]. + //This is implemented in ddcd, check there to see how to do it! + int fd; + if(sscanf(argv[3], "%d",&fd)<=0) return 0; + fprintf(stderr,"csdr: fd control mode on, fd=%d\n", fd); + int flags = fcntl(fd, F_GETFL, 0); + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + return fd; + } } return 0; } diff --git a/ddcd.cpp b/ddcd.cpp index 61ddbbc..d8e1f41 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -45,9 +45,12 @@ int in_client = 0; char ddc_method_str[100] = "td"; ddc_method_t ddc_method; pid_t main_dsp_proc; -pid_t pgrp; + int input_fd = STDIN_FILENO; //can be stdin, or the stdout of main_subprocess pid_t main_subprocess_pid = 0; +pid_t main_subprocess_pgrp = 0; +pid_t client_subprocess_pid = 0; +pid_t client_subprocess_pgrp = 0; char* buf; @@ -69,14 +72,21 @@ int proc_exists(pid_t pid) void sig_handler(int signo) { int tmpstat; + if(signo==SIGPIPE) + { + fprintf(stderr,MSG_START "SIGPIPE received.\n"); + return; + } if(signo==SIGCHLD) if( main_subprocess_pid && signo==SIGCHLD && (waitpid(main_subprocess_pid, &tmpstat, WNOHANG), 1) && !proc_exists(main_subprocess_pid) ) { fprintf(stderr,MSG_START "main_subprocess_pid exited! Exiting...\n"); } else return; - if(pgrp!=1 && pgrp!=0) //I just want to make sure that we cannot kill init or sched - killpg(pgrp, signo); + //if(pgrp!=1 && pgrp!=0) //I just want to make sure that we cannot kill init or sched + // killpg(pgrp, signo); + if( !in_client && main_subprocess_pid ) killpg2(main_subprocess_pgrp); + if( in_client && client_subprocess_pid ) killpg2(client_subprocess_pgrp); fprintf(stderr, MSG_START "signal %d caught in %s, exiting ddcd...\n", signo, (in_client)?"client":"main"); fflush(stderr); exit(0); @@ -161,7 +171,7 @@ int main(int argc, char* argv[]) sigaction(SIGINT, &sa, NULL); sigaction(SIGHUP, &sa, NULL); sigaction(SIGCHLD, &sa, NULL); - //sigaction(SIGPIPE, &sa, NULL); + sigaction(SIGPIPE, &sa, NULL); prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits struct sockaddr_in addr_host; @@ -214,8 +224,9 @@ int main(int argc, char* argv[]) } //We'll see if it is a good idea: - setpgrp(); - pgrp = getpgrp(); + //setpgrp(); + //pgrp = getpgrp(); + //It is not, because we can't catch Ctrl+C (SIGINT), as it is sent to a process group... //Start DSP subprocess from the main process if required char main_subprocess_cmd_buf[500]; @@ -236,7 +247,7 @@ int main(int argc, char* argv[]) case M_FASTDDC: sprintf(main_subprocess_cmd_buf, subprocess_args_fastddc_1, decimation, transition_bw); fprintf(stderr, MSG_START "starting main_subprocess_cmd: %s\n", main_subprocess_cmd_buf); - if(!(main_subprocess_pid = run_subprocess( main_subprocess_cmd_buf, 0, pipe_s2m ))) + if(!(main_subprocess_pid = run_subprocess( main_subprocess_cmd_buf, 0, pipe_s2m, &main_subprocess_pgrp ))) print_exit(MSG_START "couldn't start main_subprocess_cmd!\n"); close(STDIN_FILENO); // redirect stdin to the stdin of the subprocess break; @@ -322,7 +333,7 @@ int main(int argc, char* argv[]) close(clients[i]->socket); delete clients[i]; clients.erase(clients.begin()+i); - print_client(clients[i], "done closing client from main process."); + fprintf(stderr, MSG_START "done closing client from main process."); } } else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } @@ -334,9 +345,14 @@ int main(int argc, char* argv[]) return 0; } -pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out) +pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out, pid_t* pgrp) { pid_t pid = fork(); + if(*pgrp>=0) + { + setpgrp(); + *pgrp = getpgrp(); + } //fprintf(stderr, "run_subprocess :: fork-ed %d\n", pid); if(pid < 0) return 0; //fork failed if(pid == 0) @@ -376,7 +392,6 @@ void client() print_client(this_client, "client process forked."); char client_subprocess_cmd_buf[500]; - pid_t client_subprocess_pid = 0; int input_fd = this_client->pipefd[0]; prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits @@ -395,13 +410,13 @@ void client() sprintf(client_subprocess_cmd_buf, subprocess_args_fastddc_2, decimation, pipe_ctl[0], transition_bw); break; } - if(!(client_subprocess_pid = run_subprocess( client_subprocess_cmd_buf, this_client->pipefd, pipe_stdout ))) + + if(!(client_subprocess_pid = run_subprocess( client_subprocess_cmd_buf, this_client->pipefd, pipe_stdout, &client_subprocess_pgrp))) print_exit(MSG_START "couldn't start client_subprocess_cmd!\n"); fprintf(stderr, MSG_START "starting client_subprocess_cmd: %s\n", client_subprocess_cmd_buf); input_fd = pipe_stdout[0]; //we don't have to set it nonblocking fprintf(stderr, MSG_START "pipe_stdout[0] = %d\n", pipe_stdout[0]); - setpgrp(); - pgrp = getpgrp(); + write(pipe_ctl[1], "0.0\n", 4); } for(;;) { @@ -409,12 +424,17 @@ void client() if(send(this_client->socket,buf,bufsizeall,0)==-1) { print_client(this_client, "client process is exiting."); - if(client_subprocess_pid && pgrp!=1 && pgrp!=0) killpg(pgrp, SIGTERM); + if(client_subprocess_pid) killpg2(client_subprocess_pgrp); exit(0); } } } +void killpg2(pid_t pgrp) +{ + if(pgrp!=1 && pgrp!=0) killpg(pgrp, SIGTERM); +} + void error_exit(const char* why) { perror(why); diff --git a/ddcd.h b/ddcd.h index f67a4c6..f7558aa 100644 --- a/ddcd.h +++ b/ddcd.h @@ -33,9 +33,10 @@ void error_exit(const char* why); void print_exit(const char* why); void print_client(client_t* client, const char* what); int proc_exists(pid_t pid); -pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out); +pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out, pid_t* pgrp); void maxfd(int* maxfd, int fd); void sig_handler(int signo); +void killpg2(pid_t pgrp); typedef enum ddc_method_e { @@ -49,9 +50,9 @@ const char subprocess_cmd_td[] = "csdr " #else "shift_unroll_cc" #endif - " --pipe %d | csdr fir_decimate_cc %d %g"; + " --fd %d | csdr fir_decimate_cc %d %g"; -const char subprocess_args_fastddc_1[] = "csdr through %d %g"; //const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; +const char subprocess_args_fastddc_1[] = "csdr through %d %g"; //const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc %d --pipe %d %g"; const char subprocess_args_fastddc_2[] = "csdr convert_u8_f %d %d %g"; From 6952973e02db7647e8e7ff032eb5a6d061597c37 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 3 Dec 2015 00:17:46 +0100 Subject: [PATCH 29/74] Added read_socket_ctl() --- ddcd.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/ddcd.cpp b/ddcd.cpp index d8e1f41..4d27ce4 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -386,6 +386,46 @@ void client_cleanup() close(this_client->pipefd[0]); } +#define CTL_BUFSIZE 1024 + +int read_socket_ctl(int fd, char* output, int max_size) +{ + //fprintf(stderr, "doing read_socket_ctl %d\n", fd); + //if(!fd) return 0; + static char buffer[CTL_BUFSIZE]; + static int buffer_index=0; + if(buffer_index==CTL_BUFSIZE) buffer_index=0; + int bytes_read=recv(fd,buffer+buffer_index,(CTL_BUFSIZE-buffer_index)*sizeof(char), MSG_DONTWAIT); + if(bytes_read<=0) return 0; + fprintf(stderr, "recv %d\n", bytes_read); + + int prev_newline_at=0; + int last_newline_at=0; + for(int i=0;isocket, recv_cmd, CTL_BUFSIZE)) + fprintf(stderr, "read_socket_ctl: %s\n", recv_cmd); read(input_fd,buf,bufsizeall); if(send(this_client->socket,buf,bufsizeall,0)==-1) { From 35b3f631d682e7ac88f22a17688ca15c62553e70 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 3 Dec 2015 11:22:25 +0100 Subject: [PATCH 30/74] method=td is OK --- Makefile | 2 +- ddcd.cpp | 102 ++++++++++++++++++++++++++++++++++++++++++++----------- ddcd.h | 4 +++ 3 files changed, 88 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 09cd72b..c50fa11 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c fastd csdr: csdr.c libcsdr.so gcc -std=gnu99 $(PARAMS_LOOPVECT) $(PARAMS_SIMD) csdr.c $(PARAMS_LIBS) -L. -lcsdr $(PARAMS_MISC) -o csdr ddcd: ddcd.cpp libcsdr.so ddcd.h - g++ $(PARAMS_LOOPVECT) $(PARAMS_SIMD) ddcd.cpp $(PARAMS_LIBS) -L. -lcsdr $(PARAMS_MISC) -o ddcd + g++ $(PARAMS_LOOPVECT) $(PARAMS_SIMD) ddcd.cpp $(PARAMS_LIBS) -L. -lcsdr -lpthread $(PARAMS_MISC) -o ddcd arm-cross: clean-vect #note: this doesn't work since having added FFTW arm-linux-gnueabihf-gcc -std=gnu99 -O3 -fshort-double -ffast-math -dumpbase dumpvect-arm -fdump-tree-vect-details -mfloat-abi=softfp -march=armv7-a -mtune=cortex-a9 -mfpu=neon -mvectorize-with-neon-quad -Wno-unused-result -Wformat=0 $(SOURCES) -lm -o ./csdr diff --git a/ddcd.cpp b/ddcd.cpp index 4d27ce4..9538bc9 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -333,7 +333,7 @@ int main(int argc, char* argv[]) close(clients[i]->socket); delete clients[i]; clients.erase(clients.begin()+i); - fprintf(stderr, MSG_START "done closing client from main process."); + fprintf(stderr, MSG_START "done closing client from main process.\n"); } } else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } @@ -347,16 +347,24 @@ int main(int argc, char* argv[]) pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out, pid_t* pgrp) { + /*char sem_name[101]; + snprintf(sem_name,100,"ddcd_sem_%d",getpid()); + sem_t mysem; + if(sem_init(&mysem, 1, 1)==-1) error_exit("failed to sem_init() in run_subprocess()"); + fprintf(stderr, "sem_waiting\n"); + if(sem_wait(&mysem)==-1) error_exit("the first sem_wait() failed in run_subprocess()"); + fprintf(stderr, "sem_waited\n"); + */ + int syncpipe[2]; + if(pipe(syncpipe)==-1) error_exit("failed to create pipe()"); pid_t pid = fork(); - if(*pgrp>=0) - { - setpgrp(); - *pgrp = getpgrp(); - } - //fprintf(stderr, "run_subprocess :: fork-ed %d\n", pid); + if(pid < 0) return 0; //fork failed if(pid == 0) { + setpgrp(); + write(syncpipe[1], " ", 1); + //if(sem_post(&mysem)==-1) error_exit("failed to sem_post() in run_subprocess()"); //We're the subprocess //fprintf(stderr, "run_subprocess :: execl\n"); //if(fcntl(pipe_in[1], F_SETPIPE_SZ, pipe_max_size) == -1) perror("Failed to F_SETPIPE_SZ in run_subprocess()"); @@ -373,7 +381,15 @@ pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out, pid_t* pgrp) execl("/bin/bash","bash","-c",cmd, (char*)0); error_exit(MSG_START "run_subprocess failed to execute command"); } - else return pid; + else + { + //if(sem_wait(&mysem)==-1) error_exit("the second sem_wait() failed in run_subprocess()"); + int synctemp; + read(syncpipe[0], &synctemp, 1); + *pgrp = getpgid(pid); + fprintf(stderr, MSG_START "run_subprocess pgid returned = %d\n", *pgrp); + return pid; + } } void print_client(client_t* client, const char* what) @@ -381,11 +397,6 @@ void print_client(client_t* client, const char* what) fprintf(stderr,MSG_START "(client %s:%d) %s\n", inet_ntoa(client->addr.sin_addr), client->addr.sin_port, what); } -void client_cleanup() -{ - close(this_client->pipefd[0]); -} - #define CTL_BUFSIZE 1024 int read_socket_ctl(int fd, char* output, int max_size) @@ -397,7 +408,7 @@ int read_socket_ctl(int fd, char* output, int max_size) if(buffer_index==CTL_BUFSIZE) buffer_index=0; int bytes_read=recv(fd,buffer+buffer_index,(CTL_BUFSIZE-buffer_index)*sizeof(char), MSG_DONTWAIT); if(bytes_read<=0) return 0; - fprintf(stderr, "recv %d\n", bytes_read); + //fprintf(stderr, "recv %d\n", bytes_read); int prev_newline_at=0; int last_newline_at=0; @@ -425,6 +436,26 @@ int read_socket_ctl(int fd, char* output, int max_size) } } +int ctl_get_arg(char* input, const char* cmd, const char* format, ...) +{ + int retval=0; + int cmdlen=strlen(cmd); + if(input[cmdlen]=='=') + { + //fprintf(stderr, "cga found=\n"); + if(input[cmdlen]=0, !strcmp(input,cmd)) + { + //fprintf(stderr, "cga foundokay\n"); + va_list vl; + va_start(vl,format); + retval=vsscanf(input+cmdlen+1,format,vl); + va_end(vl); + } + input[cmdlen]='='; + } + //fprintf(stderr, "cga retval %d\n", retval); + return retval; +} void client() { @@ -433,12 +464,13 @@ void client() char client_subprocess_cmd_buf[500]; int input_fd = this_client->pipefd[0]; + int pipe_ctl[2], pipe_stdout[2]; prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits if(decimation!=1) { - int pipe_ctl[2], pipe_stdout[2]; + if(pipe(pipe_ctl)==-1) error_exit(MSG_START "cannot open new pipe() for the client subprocess"); if(pipe(pipe_stdout)==-1) error_exit(MSG_START "cannot open new pipe() for the client subprocess"); switch(ddc_method) @@ -459,14 +491,45 @@ void client() write(pipe_ctl[1], "0.0\n", 4); } char recv_cmd[CTL_BUFSIZE]; + char temps[CTL_BUFSIZE*2]; + int tempi; + float tempf; + for(;;) { while(read_socket_ctl(this_client->socket, recv_cmd, CTL_BUFSIZE)) - fprintf(stderr, "read_socket_ctl: %s\n", recv_cmd); - read(input_fd,buf,bufsizeall); - if(send(this_client->socket,buf,bufsizeall,0)==-1) { - print_client(this_client, "client process is exiting."); + sprintf(temps, "read_socket_ctl: %s", recv_cmd); + print_client(this_client, temps); + if(ctl_get_arg(recv_cmd, "bypass", "%d", &tempi)) + { + if(tempi==1 && client_subprocess_pid) + { + //print_client(this_client, "suspending client_subprocess_pgrp...\n"); + //fprintf(stderr, "client_subprocess_pgrp = %d\n", client_subprocess_pgrp); + //killpg(client_subprocess_pgrp, SIGTSTP); + //while(proc_exists(client_subprocess_pid)) usleep(10000); + //print_client(this_client, "done killing client_subprocess_pid.\n"); + input_fd=this_client->pipefd[0]; //by doing this, we don't read from pipe_stdout[0] anymore, so that csdr stops doing anything, and also doesn't read anymore from the input: we get the whole I/Q stream! + } + if(tempi==0 && client_subprocess_pid) + { + input_fd=pipe_stdout[0]; + } + + } + if(ctl_get_arg(recv_cmd, "shift", "%g", &tempf)) + { + tempi=sprintf(temps, "%g\n", tempf); + write(pipe_ctl[1], temps, tempi); + fsync(pipe_ctl[1]); + } + } + int nread = read(input_fd,buf,bufsizeall); + if(nread<=0) continue; + if(send(this_client->socket,buf,nread,0)==-1) + { + print_client(this_client, "client process is exiting.\n"); if(client_subprocess_pid) killpg2(client_subprocess_pgrp); exit(0); } @@ -475,6 +538,7 @@ void client() void killpg2(pid_t pgrp) { + //fprintf(stderr, MSG_START "killpg2: %d\n", pgrp); if(pgrp!=1 && pgrp!=0) killpg(pgrp, SIGTERM); } diff --git a/ddcd.h b/ddcd.h index f7558aa..9edd913 100644 --- a/ddcd.h +++ b/ddcd.h @@ -16,6 +16,9 @@ #include #include #include +#include +#include +#include typedef struct client_s { @@ -37,6 +40,7 @@ pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out, pid_t* pgrp); void maxfd(int* maxfd, int fd); void sig_handler(int signo); void killpg2(pid_t pgrp); +int ctl_get_arg(char* input, const char* cmd, const char* format, ...); typedef enum ddc_method_e { From 5fede66e27eb270075e09c9bb5122c5c80acbe0a Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 3 Dec 2015 17:08:30 +0100 Subject: [PATCH 31/74] fastddc_inv_cc is now controllable via --fd and --fifo. --- csdr.c | 35 +++++++++++++++++++++++++++-------- ddcd.cpp | 2 +- ddcd.h | 8 ++++---- grc_tests/test_fastddc.grc | 10 +++++----- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/csdr.c b/csdr.c index f6e4799..ba87564 100644 --- a/csdr.c +++ b/csdr.c @@ -1808,22 +1808,38 @@ int main(int argc, char *argv[]) } } - if( !strcmp(argv[1],"fastddc_inv_cc") ) // [transition_bw [window]] + if( !strcmp(argv[1],"fastddc_inv_cc") ) // [transition_bw [window]] { - int decimation; - if(argc<=2) return badsyntax("need required parameter (decimation)"); - sscanf(argv[2],"%d",&decimation); - float shift_rate; - if(argc>3) sscanf(argv[3],"%g",&shift_rate); + int plusarg=0; + + int fd; + if(fd=init_fifo(argc,argv)) + { + while(!read_fifo_ctl(fd,"%g\n",&shift_rate)) usleep(10000); + plusarg=1; + } + else + { + if(argc<=2) return badsyntax("need required parameter (rate)"); + sscanf(argv[2],"%g",&shift_rate); + } + + int decimation; + if(argc<=3+plusarg) return badsyntax("need required parameter (decimation)"); + sscanf(argv[3+plusarg],"%d",&decimation); + //fprintf(stderr, "dec=%d %d\n", decimation); float transition_bw = 0.05; - if(argc>4) sscanf(argv[4],"%g",&transition_bw); + if(argc>4+plusarg) sscanf(argv[4+plusarg],"%g",&transition_bw); window_t window = WINDOW_DEFAULT; - if(argc>5) window=firdes_get_window_from_string(argv[5]); + if(argc>5+plusarg) window=firdes_get_window_from_string(argv[5+plusarg]); else fprintf(stderr,"fastddc_apply_cc: window = %s\n",firdes_get_string_from_window(window)); + for(;;) + { + fastddc_t ddc; if(fastddc_init(&ddc, transition_bw, decimation, shift_rate)) { badsyntax("error in fastddc_init()"); return 1; } fastddc_print(&ddc,"fastddc_inv_cc"); @@ -1864,6 +1880,9 @@ int main(int argc, char *argv[]) fwrite(output, sizeof(complexf), shift_stat.output_size, stdout); //fprintf(stderr, "ss os = %d\n", shift_stat.output_size); TRY_YIELD; + if(read_fifo_ctl(fd,"%g\n",&shift_rate)) break; + } + } } diff --git a/ddcd.cpp b/ddcd.cpp index 9538bc9..11a17ac 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -479,7 +479,7 @@ void client() sprintf(client_subprocess_cmd_buf, subprocess_cmd_td, pipe_ctl[0], decimation, transition_bw); break; case M_FASTDDC: - sprintf(client_subprocess_cmd_buf, subprocess_args_fastddc_2, decimation, pipe_ctl[0], transition_bw); + sprintf(client_subprocess_cmd_buf, subprocess_args_fastddc_2, pipe_ctl[0], decimation, transition_bw); break; } diff --git a/ddcd.h b/ddcd.h index 9edd913..af4cfb5 100644 --- a/ddcd.h +++ b/ddcd.h @@ -56,7 +56,7 @@ const char subprocess_cmd_td[] = "csdr " #endif " --fd %d | csdr fir_decimate_cc %d %g"; -//const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; -const char subprocess_args_fastddc_1[] = "csdr through %d %g"; -//const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc %d --pipe %d %g"; -const char subprocess_args_fastddc_2[] = "csdr convert_u8_f %d %d %g"; +const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; +//const char subprocess_args_fastddc_1[] = "csdr through %d %g"; +const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc --fd %d %d %g"; +//const char subprocess_args_fastddc_2[] = "csdr convert_u8_f %d %d %g"; diff --git a/grc_tests/test_fastddc.grc b/grc_tests/test_fastddc.grc index 829a336..14acb88 100644 --- a/grc_tests/test_fastddc.grc +++ b/grc_tests/test_fastddc.grc @@ -257,7 +257,7 @@ _coordinate - (112, 875) + (112, 675) _rotation @@ -375,7 +375,7 @@ _coordinate - (136, 763) + (368, 691) _rotation @@ -465,7 +465,7 @@ commandline - csdr fastddc_fwd_cc %d | csdr fastddc_inv_cc %d 0.4"%(decimation,decimation)+" + csdr fastddc_fwd_cc %d | csdr fastddc_inv_cc 0.4 %d"%(decimation,decimation)+" comment @@ -690,7 +690,7 @@ _coordinate - (616, 379) + (616, 291) _rotation @@ -789,7 +789,7 @@ _coordinate - (400, 691) + (576, 619) _rotation From e71c33565e0f16a570fac18fe315addc1cf616b5 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Fri, 18 Dec 2015 07:51:10 +0000 Subject: [PATCH 32/74] Added fir_decimate_cc to test200. --- test200.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test200.c b/test200.c index d2da851..0a89708 100644 --- a/test200.c +++ b/test200.c @@ -41,6 +41,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define T_BUFSIZE (1024*1024/4) #define T_N (200) +#define T_TAPS (1023) +#define T_DECFACT (200) int main() { @@ -58,10 +60,21 @@ int main() qof(buf_c,i)=buf_u8[2*i+1]/128.0; } + + float* taps_f = (float*)malloc(sizeof(float)*T_TAPS); + firdes_lowpass_f(taps_f, T_TAPS, 1.0f/T_DECFACT, WINDOW_DEFAULT); + struct timespec start_time, end_time; fprintf(stderr,"Starting tests of processing %d samples...\n", T_BUFSIZE*T_N); + //fir_decimate_cc + clock_gettime(CLOCK_MONOTONIC_RAW, &start_time); + for(int i=0;i Date: Tue, 12 Jan 2016 23:01:36 +0100 Subject: [PATCH 33/74] Started writing a new DDCD based on pthreads. --- Makefile | 2 +- ddcd.cpp | 447 ---------------------------------------- ddcd.h | 62 ++---- ddcd_old.cpp | 560 +++++++++++++++++++++++++++++++++++++++++++++++++++ ddcd_old.h | 62 ++++++ 5 files changed, 636 insertions(+), 497 deletions(-) create mode 100644 ddcd_old.cpp create mode 100644 ddcd_old.h diff --git a/Makefile b/Makefile index c50fa11..8294e8f 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ FFTW_PACKAGE = fftw-3.3.3 .PHONY: clean-vect clean all: csdr ddcd -libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c fastddc.c *.h +libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c fastddc.c fastddc.h fft_fftw.h fft_rpi.h ima_adpcm.h libcsdr_gpl.h libcsdr.h predefined.h @echo NOTE: you may have to manually edit Makefile to optimize for your CPU \(especially if you compile on ARM, please edit PARAMS_NEON\). @echo Auto-detected optimization parameters: $(PARAMS_SIMD) @echo diff --git a/ddcd.cpp b/ddcd.cpp index 11a17ac..5222022 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -30,75 +30,17 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ddcd.h" - -#define SOFTWARE_NAME "ddcd" -#define MSG_START SOFTWARE_NAME ": " - int host_port = 0; char host_address[100] = "127.0.0.1"; int decimation = 0; float transition_bw = 0.05; int bufsize = 1024; -int bufsizeall; -int pipe_max_size; -int in_client = 0; char ddc_method_str[100] = "td"; ddc_method_t ddc_method; -pid_t main_dsp_proc; - -int input_fd = STDIN_FILENO; //can be stdin, or the stdout of main_subprocess -pid_t main_subprocess_pid = 0; -pid_t main_subprocess_pgrp = 0; -pid_t client_subprocess_pid = 0; -pid_t client_subprocess_pgrp = 0; - -char* buf; - -int set_nonblocking(int fd) -{ - int flagtmp; - if((flagtmp = fcntl(fd, F_GETFL))!=-1) - if((flagtmp = fcntl(fd, F_SETFL, flagtmp|O_NONBLOCK))!=-1) - return 0; - return 1; -} - -int proc_exists(pid_t pid) -{ - if(pid==0 || pid==1) return 1; - return kill(pid, 0) != -1; -} - -void sig_handler(int signo) -{ - int tmpstat; - if(signo==SIGPIPE) - { - fprintf(stderr,MSG_START "SIGPIPE received.\n"); - return; - } - if(signo==SIGCHLD) - if( main_subprocess_pid && signo==SIGCHLD && (waitpid(main_subprocess_pid, &tmpstat, WNOHANG), 1) && !proc_exists(main_subprocess_pid) ) - { - fprintf(stderr,MSG_START "main_subprocess_pid exited! Exiting...\n"); - } - else return; - //if(pgrp!=1 && pgrp!=0) //I just want to make sure that we cannot kill init or sched - // killpg(pgrp, signo); - if( !in_client && main_subprocess_pid ) killpg2(main_subprocess_pgrp); - if( in_client && client_subprocess_pid ) killpg2(client_subprocess_pgrp); - fprintf(stderr, MSG_START "signal %d caught in %s, exiting ddcd...\n", signo, (in_client)?"client":"main"); - fflush(stderr); - exit(0); -} - -client_t* this_client; int main(int argc, char* argv[]) { int c; - fd_set select_fds; - for(;;) { int option_index = 0; @@ -161,391 +103,7 @@ int main(int argc, char* argv[]) } else print_exit(MSG_START "invalid parameter given to --method.\n"); - //set signals - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = sig_handler; - sigaction(SIGTERM, &sa, NULL); - sigaction(SIGKILL, &sa, NULL); - sigaction(SIGQUIT, &sa, NULL); - sigaction(SIGINT, &sa, NULL); - sigaction(SIGHUP, &sa, NULL); - sigaction(SIGCHLD, &sa, NULL); - sigaction(SIGPIPE, &sa, NULL); - prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits - struct sockaddr_in addr_host; - int listen_socket; - std::vector clients; - clients.reserve(100); - listen_socket=socket(AF_INET,SOCK_STREAM,0); - - int sockopt = 1; - if( setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&sockopt, sizeof(sockopt)) == -1 ) - error_exit(MSG_START "cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 - - memset(&addr_host,'0',sizeof(addr_host)); - addr_host.sin_family=AF_INET; - addr_host.sin_port=htons(host_port); - addr_host.sin_addr.s_addr = INADDR_ANY; - - if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) - error_exit(MSG_START "invalid host address"); - - if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) - error_exit(MSG_START "cannot bind() address to the socket"); - - if( listen(listen_socket, 10) == -1 ) - error_exit(MSG_START "cannot listen() on socket"); - - fprintf(stderr,MSG_START "listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); - - struct sockaddr_in addr_cli; - socklen_t addr_cli_len = sizeof(addr_cli); - int new_socket; - - bufsizeall = bufsize*sizeof(char); - buf = (char*)malloc(bufsizeall); - - FILE* tempfile = fopen("/proc/sys/fs/pipe-max-size","r"); - if(!tempfile) - { - perror(MSG_START "cannot read /proc/sys/fs/pipe-max-size"); - } - else - { - char pipe_max_size_str[100]; - int tfread = fread(pipe_max_size_str, 1, 100, tempfile); - pipe_max_size_str[tfread]='\0'; - pipe_max_size = atoi(pipe_max_size_str); - //fprintf(stderr, MSG_START "note: pipe_max_size = %d\n", pipe_max_size); - //if(pipe_max_size>4096 && fcntl(STDIN_FILENO, F_SETPIPE_SZ, pipe_max_size)==-1) - // perror("failed to fcntl(STDIN_FILENO, F_SETPIPE_SZ, ...)"); - } - - //We'll see if it is a good idea: - //setpgrp(); - //pgrp = getpgrp(); - //It is not, because we can't catch Ctrl+C (SIGINT), as it is sent to a process group... - - //Start DSP subprocess from the main process if required - char main_subprocess_cmd_buf[500]; - - - int pipe_m2s_ctl[2]; //main to subprocess :: control channel - int pipe_s2m[2]; //subprocess to main - - if(pipe(pipe_m2s_ctl)) error_exit(MSG_START "couldn't create pipe_m2s_ctl"); - if(pipe(pipe_s2m)) error_exit(MSG_START "couldn't create pipe_s2m"); - - if(decimation!=1) - { - switch(ddc_method) - { - case M_TD: - break; - case M_FASTDDC: - sprintf(main_subprocess_cmd_buf, subprocess_args_fastddc_1, decimation, transition_bw); - fprintf(stderr, MSG_START "starting main_subprocess_cmd: %s\n", main_subprocess_cmd_buf); - if(!(main_subprocess_pid = run_subprocess( main_subprocess_cmd_buf, 0, pipe_s2m, &main_subprocess_pgrp ))) - print_exit(MSG_START "couldn't start main_subprocess_cmd!\n"); - close(STDIN_FILENO); // redirect stdin to the stdin of the subprocess - break; - } - } - - int highfd = 0; - FD_ZERO(&select_fds); - FD_SET(listen_socket, &select_fds); - maxfd(&highfd, listen_socket); - if(main_subprocess_pid) input_fd = pipe_s2m[0]; //else STDIN_FILENO - FD_SET(input_fd, &select_fds); - maxfd(&highfd, input_fd); - - //Set stdin and listen_socket to non-blocking - if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) //don't do it before subprocess fork! - error_exit(MSG_START "cannot set_nonblocking()"); - - for(;;) - { - //Let's wait until there is any new data to read, or any new connection! - select(highfd, &select_fds, NULL, NULL, NULL); - - //Is there a new client connection? - if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) - { - this_client = new client_t; - this_client->error = 0; - memcpy(&this_client->addr, &addr_cli, sizeof(this_client->addr)); - this_client->socket = new_socket; - if(pipe(this_client->pipefd) == -1) - { - perror(MSG_START "cannot open new pipe() for the client"); - continue; - } - if(fcntl(this_client->pipefd[1], F_SETPIPE_SZ, pipe_max_size) == -1) - perror("failed to F_SETPIPE_SZ for the client pipe"); - if(this_client->pid = fork()) - { - //We're the parent - set_nonblocking(this_client->pipefd[1]); - clients.push_back(this_client); - fprintf(stderr, MSG_START "client pid: %d\n", this_client->pid); - } - else - { - //We're the client - client(); - return 1; - } - } - - int retval = read(input_fd, buf, bufsizeall); - if(retval==0) - { - //end of input stream, close clients and exit - } - else if(retval != -1) - { - for (int i=0; ipipefd[1], buf, retval)==-1) - { - - if(!clients[i]->error) - { - print_client(clients[i], "lost buffer, failed to write pipe."); - clients[i]->error=1; - } - //fprintf(stderr, MSG_START "errno is %d\n", errno); //usually 11 - //int wpstatus; - //int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); - //fprintf(stderr, MSG_START "pid is %d\n",clients[i]->pid); - //perror("somethings wrong"); - //if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); - //else if(wpresult == 0) - waitpid(clients[i]->pid, NULL, WNOHANG); - if(!proc_exists(clients[i]->pid)) - { - //Client exited! - print_client(clients[i], "closing client from main process."); - close(clients[i]->pipefd[1]); - close(clients[i]->socket); - delete clients[i]; - clients.erase(clients.begin()+i); - fprintf(stderr, MSG_START "done closing client from main process.\n"); - } - } - else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } - } - } - //TODO: at the end, server closes pipefd[1] for client - } - - return 0; -} - -pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out, pid_t* pgrp) -{ - /*char sem_name[101]; - snprintf(sem_name,100,"ddcd_sem_%d",getpid()); - sem_t mysem; - if(sem_init(&mysem, 1, 1)==-1) error_exit("failed to sem_init() in run_subprocess()"); - fprintf(stderr, "sem_waiting\n"); - if(sem_wait(&mysem)==-1) error_exit("the first sem_wait() failed in run_subprocess()"); - fprintf(stderr, "sem_waited\n"); - */ - int syncpipe[2]; - if(pipe(syncpipe)==-1) error_exit("failed to create pipe()"); - pid_t pid = fork(); - - if(pid < 0) return 0; //fork failed - if(pid == 0) - { - setpgrp(); - write(syncpipe[1], " ", 1); - //if(sem_post(&mysem)==-1) error_exit("failed to sem_post() in run_subprocess()"); - //We're the subprocess - //fprintf(stderr, "run_subprocess :: execl\n"); - //if(fcntl(pipe_in[1], F_SETPIPE_SZ, pipe_max_size) == -1) perror("Failed to F_SETPIPE_SZ in run_subprocess()"); - if(pipe_in) - { - close(pipe_in[1]); - dup2(pipe_in[0], STDIN_FILENO); - } - if(pipe_out) - { - close(pipe_out[0]); - dup2(pipe_out[1], STDOUT_FILENO); - } - execl("/bin/bash","bash","-c",cmd, (char*)0); - error_exit(MSG_START "run_subprocess failed to execute command"); - } - else - { - //if(sem_wait(&mysem)==-1) error_exit("the second sem_wait() failed in run_subprocess()"); - int synctemp; - read(syncpipe[0], &synctemp, 1); - *pgrp = getpgid(pid); - fprintf(stderr, MSG_START "run_subprocess pgid returned = %d\n", *pgrp); - return pid; - } -} - -void print_client(client_t* client, const char* what) -{ - fprintf(stderr,MSG_START "(client %s:%d) %s\n", inet_ntoa(client->addr.sin_addr), client->addr.sin_port, what); -} - -#define CTL_BUFSIZE 1024 - -int read_socket_ctl(int fd, char* output, int max_size) -{ - //fprintf(stderr, "doing read_socket_ctl %d\n", fd); - //if(!fd) return 0; - static char buffer[CTL_BUFSIZE]; - static int buffer_index=0; - if(buffer_index==CTL_BUFSIZE) buffer_index=0; - int bytes_read=recv(fd,buffer+buffer_index,(CTL_BUFSIZE-buffer_index)*sizeof(char), MSG_DONTWAIT); - if(bytes_read<=0) return 0; - //fprintf(stderr, "recv %d\n", bytes_read); - - int prev_newline_at=0; - int last_newline_at=0; - for(int i=0;ipipefd[0]; - int pipe_ctl[2], pipe_stdout[2]; - - prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits - - if(decimation!=1) - { - - if(pipe(pipe_ctl)==-1) error_exit(MSG_START "cannot open new pipe() for the client subprocess"); - if(pipe(pipe_stdout)==-1) error_exit(MSG_START "cannot open new pipe() for the client subprocess"); - switch(ddc_method) - { - case M_TD: - sprintf(client_subprocess_cmd_buf, subprocess_cmd_td, pipe_ctl[0], decimation, transition_bw); - break; - case M_FASTDDC: - sprintf(client_subprocess_cmd_buf, subprocess_args_fastddc_2, pipe_ctl[0], decimation, transition_bw); - break; - } - - if(!(client_subprocess_pid = run_subprocess( client_subprocess_cmd_buf, this_client->pipefd, pipe_stdout, &client_subprocess_pgrp))) - print_exit(MSG_START "couldn't start client_subprocess_cmd!\n"); - fprintf(stderr, MSG_START "starting client_subprocess_cmd: %s\n", client_subprocess_cmd_buf); - input_fd = pipe_stdout[0]; //we don't have to set it nonblocking - fprintf(stderr, MSG_START "pipe_stdout[0] = %d\n", pipe_stdout[0]); - write(pipe_ctl[1], "0.0\n", 4); - } - char recv_cmd[CTL_BUFSIZE]; - char temps[CTL_BUFSIZE*2]; - int tempi; - float tempf; - - for(;;) - { - while(read_socket_ctl(this_client->socket, recv_cmd, CTL_BUFSIZE)) - { - sprintf(temps, "read_socket_ctl: %s", recv_cmd); - print_client(this_client, temps); - if(ctl_get_arg(recv_cmd, "bypass", "%d", &tempi)) - { - if(tempi==1 && client_subprocess_pid) - { - //print_client(this_client, "suspending client_subprocess_pgrp...\n"); - //fprintf(stderr, "client_subprocess_pgrp = %d\n", client_subprocess_pgrp); - //killpg(client_subprocess_pgrp, SIGTSTP); - //while(proc_exists(client_subprocess_pid)) usleep(10000); - //print_client(this_client, "done killing client_subprocess_pid.\n"); - input_fd=this_client->pipefd[0]; //by doing this, we don't read from pipe_stdout[0] anymore, so that csdr stops doing anything, and also doesn't read anymore from the input: we get the whole I/Q stream! - } - if(tempi==0 && client_subprocess_pid) - { - input_fd=pipe_stdout[0]; - } - - } - if(ctl_get_arg(recv_cmd, "shift", "%g", &tempf)) - { - tempi=sprintf(temps, "%g\n", tempf); - write(pipe_ctl[1], temps, tempi); - fsync(pipe_ctl[1]); - } - } - int nread = read(input_fd,buf,bufsizeall); - if(nread<=0) continue; - if(send(this_client->socket,buf,nread,0)==-1) - { - print_client(this_client, "client process is exiting.\n"); - if(client_subprocess_pid) killpg2(client_subprocess_pgrp); - exit(0); - } - } -} - -void killpg2(pid_t pgrp) -{ - //fprintf(stderr, MSG_START "killpg2: %d\n", pgrp); - if(pgrp!=1 && pgrp!=0) killpg(pgrp, SIGTERM); -} - -void error_exit(const char* why) -{ - perror(why); - exit(1); } void print_exit(const char* why) @@ -553,8 +111,3 @@ void print_exit(const char* why) fprintf(stderr, "%s", why); exit(1); } - -void maxfd(int* maxfd, int fd) -{ - if(fd>=*maxfd) *maxfd=fd+1; -} diff --git a/ddcd.h b/ddcd.h index af4cfb5..845cc33 100644 --- a/ddcd.h +++ b/ddcd.h @@ -1,46 +1,12 @@ #pragma once -#include #include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include -typedef struct client_s -{ - struct sockaddr_in addr; - int socket; - pid_t pid; - int pipefd[2]; - int error; - pid_t dsp_proc; -} client_t; - - -void client(); -void error_exit(const char* why); -void print_exit(const char* why); -void print_client(client_t* client, const char* what); -int proc_exists(pid_t pid); -pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out, pid_t* pgrp); -void maxfd(int* maxfd, int fd); -void sig_handler(int signo); -void killpg2(pid_t pgrp); -int ctl_get_arg(char* input, const char* cmd, const char* format, ...); +#define SOFTWARE_NAME "ddcd" +#define MSG_START SOFTWARE_NAME ": " typedef enum ddc_method_e { @@ -48,15 +14,13 @@ typedef enum ddc_method_e M_FASTDDC } ddc_method_t; -const char subprocess_cmd_td[] = "csdr " -#ifdef NEON_OPTS - "shift_addfast_cc" -#else - "shift_unroll_cc" -#endif - " --fd %d | csdr fir_decimate_cc %d %g"; +void print_exit(const char* why); + +typedef struct client_s +{ + struct sockaddr_in addr; + int socket; + int error; + pthread_t thread; +} client_t; -const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; -//const char subprocess_args_fastddc_1[] = "csdr through %d %g"; -const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc --fd %d %d %g"; -//const char subprocess_args_fastddc_2[] = "csdr convert_u8_f %d %d %g"; diff --git a/ddcd_old.cpp b/ddcd_old.cpp new file mode 100644 index 0000000..11a17ac --- /dev/null +++ b/ddcd_old.cpp @@ -0,0 +1,560 @@ +/* +This software is part of libcsdr, a set of simple DSP routines for +Software Defined Radio. + +Copyright (c) 2014, Andras Retzler +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL ANDRAS RETZLER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "ddcd.h" + + +#define SOFTWARE_NAME "ddcd" +#define MSG_START SOFTWARE_NAME ": " + +int host_port = 0; +char host_address[100] = "127.0.0.1"; +int decimation = 0; +float transition_bw = 0.05; +int bufsize = 1024; +int bufsizeall; +int pipe_max_size; +int in_client = 0; +char ddc_method_str[100] = "td"; +ddc_method_t ddc_method; +pid_t main_dsp_proc; + +int input_fd = STDIN_FILENO; //can be stdin, or the stdout of main_subprocess +pid_t main_subprocess_pid = 0; +pid_t main_subprocess_pgrp = 0; +pid_t client_subprocess_pid = 0; +pid_t client_subprocess_pgrp = 0; + +char* buf; + +int set_nonblocking(int fd) +{ + int flagtmp; + if((flagtmp = fcntl(fd, F_GETFL))!=-1) + if((flagtmp = fcntl(fd, F_SETFL, flagtmp|O_NONBLOCK))!=-1) + return 0; + return 1; +} + +int proc_exists(pid_t pid) +{ + if(pid==0 || pid==1) return 1; + return kill(pid, 0) != -1; +} + +void sig_handler(int signo) +{ + int tmpstat; + if(signo==SIGPIPE) + { + fprintf(stderr,MSG_START "SIGPIPE received.\n"); + return; + } + if(signo==SIGCHLD) + if( main_subprocess_pid && signo==SIGCHLD && (waitpid(main_subprocess_pid, &tmpstat, WNOHANG), 1) && !proc_exists(main_subprocess_pid) ) + { + fprintf(stderr,MSG_START "main_subprocess_pid exited! Exiting...\n"); + } + else return; + //if(pgrp!=1 && pgrp!=0) //I just want to make sure that we cannot kill init or sched + // killpg(pgrp, signo); + if( !in_client && main_subprocess_pid ) killpg2(main_subprocess_pgrp); + if( in_client && client_subprocess_pid ) killpg2(client_subprocess_pgrp); + fprintf(stderr, MSG_START "signal %d caught in %s, exiting ddcd...\n", signo, (in_client)?"client":"main"); + fflush(stderr); + exit(0); +} + +client_t* this_client; + +int main(int argc, char* argv[]) +{ + int c; + fd_set select_fds; + + for(;;) + { + int option_index = 0; + static struct option long_options[] = { + {"port", required_argument, 0, 'p' }, + {"address", required_argument, 0, 'a' }, + {"decimation", required_argument, 0, 'd' }, + {"bufsize", required_argument, 0, 'b' }, + {"method", required_argument, 0, 'm' }, + {"transition", required_argument, 0, 't' } + }; + c = getopt_long(argc, argv, "p:a:d:b:m:t:", long_options, &option_index); + if(c==-1) break; + switch (c) + { + case 'a': + host_address[100-1]=0; + strncpy(host_address,optarg,100-1); + break; + case 'p': + host_port=atoi(optarg); + break; + case 'd': + decimation=atoi(optarg); + break; + case 'b': + bufsize=atoi(optarg); + break; + case 'm': + ddc_method_str[100-1]=0; + strncpy(ddc_method_str,optarg,100-1); + break; + case 't': + sscanf(optarg,"%g",&transition_bw); + break; + case 0: + case '?': + case ':': + default:; + print_exit(MSG_START "error in getopt_long()\n"); + } + } + + if(!decimation) print_exit(MSG_START "missing required command line argument, --decimation.\n"); + if(!host_port) print_exit(MSG_START "missing required command line argument, --port.\n"); + if(decimation<0) print_exit(MSG_START "invalid value for --decimation (should be >0).\n"); + if(decimation==1) fprintf(stderr, MSG_START "decimation = 1, just copying raw samples.\n"); + if(transition_bw<0||transition_bw>0.5) print_exit(MSG_START "invalid value for --transition (should be between 0 and 0.5).\n"); + + if(decimation==1); //don't do anything then + else if(!strcmp(ddc_method_str,"td")) + { + ddc_method = M_TD; + fprintf(stderr, MSG_START "method is M_TD (default).\n"); + } + else if (!strcmp(ddc_method_str,"fastddc")) + { + ddc_method = M_FASTDDC; + fprintf(stderr, MSG_START "method is M_FASTDDC.\n"); + } + else print_exit(MSG_START "invalid parameter given to --method.\n"); + + //set signals + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sig_handler; + sigaction(SIGTERM, &sa, NULL); + sigaction(SIGKILL, &sa, NULL); + sigaction(SIGQUIT, &sa, NULL); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGHUP, &sa, NULL); + sigaction(SIGCHLD, &sa, NULL); + sigaction(SIGPIPE, &sa, NULL); + prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits + + struct sockaddr_in addr_host; + int listen_socket; + std::vector clients; + clients.reserve(100); + listen_socket=socket(AF_INET,SOCK_STREAM,0); + + int sockopt = 1; + if( setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&sockopt, sizeof(sockopt)) == -1 ) + error_exit(MSG_START "cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 + + memset(&addr_host,'0',sizeof(addr_host)); + addr_host.sin_family=AF_INET; + addr_host.sin_port=htons(host_port); + addr_host.sin_addr.s_addr = INADDR_ANY; + + if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) + error_exit(MSG_START "invalid host address"); + + if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) + error_exit(MSG_START "cannot bind() address to the socket"); + + if( listen(listen_socket, 10) == -1 ) + error_exit(MSG_START "cannot listen() on socket"); + + fprintf(stderr,MSG_START "listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); + + struct sockaddr_in addr_cli; + socklen_t addr_cli_len = sizeof(addr_cli); + int new_socket; + + bufsizeall = bufsize*sizeof(char); + buf = (char*)malloc(bufsizeall); + + FILE* tempfile = fopen("/proc/sys/fs/pipe-max-size","r"); + if(!tempfile) + { + perror(MSG_START "cannot read /proc/sys/fs/pipe-max-size"); + } + else + { + char pipe_max_size_str[100]; + int tfread = fread(pipe_max_size_str, 1, 100, tempfile); + pipe_max_size_str[tfread]='\0'; + pipe_max_size = atoi(pipe_max_size_str); + //fprintf(stderr, MSG_START "note: pipe_max_size = %d\n", pipe_max_size); + //if(pipe_max_size>4096 && fcntl(STDIN_FILENO, F_SETPIPE_SZ, pipe_max_size)==-1) + // perror("failed to fcntl(STDIN_FILENO, F_SETPIPE_SZ, ...)"); + } + + //We'll see if it is a good idea: + //setpgrp(); + //pgrp = getpgrp(); + //It is not, because we can't catch Ctrl+C (SIGINT), as it is sent to a process group... + + //Start DSP subprocess from the main process if required + char main_subprocess_cmd_buf[500]; + + + int pipe_m2s_ctl[2]; //main to subprocess :: control channel + int pipe_s2m[2]; //subprocess to main + + if(pipe(pipe_m2s_ctl)) error_exit(MSG_START "couldn't create pipe_m2s_ctl"); + if(pipe(pipe_s2m)) error_exit(MSG_START "couldn't create pipe_s2m"); + + if(decimation!=1) + { + switch(ddc_method) + { + case M_TD: + break; + case M_FASTDDC: + sprintf(main_subprocess_cmd_buf, subprocess_args_fastddc_1, decimation, transition_bw); + fprintf(stderr, MSG_START "starting main_subprocess_cmd: %s\n", main_subprocess_cmd_buf); + if(!(main_subprocess_pid = run_subprocess( main_subprocess_cmd_buf, 0, pipe_s2m, &main_subprocess_pgrp ))) + print_exit(MSG_START "couldn't start main_subprocess_cmd!\n"); + close(STDIN_FILENO); // redirect stdin to the stdin of the subprocess + break; + } + } + + int highfd = 0; + FD_ZERO(&select_fds); + FD_SET(listen_socket, &select_fds); + maxfd(&highfd, listen_socket); + if(main_subprocess_pid) input_fd = pipe_s2m[0]; //else STDIN_FILENO + FD_SET(input_fd, &select_fds); + maxfd(&highfd, input_fd); + + //Set stdin and listen_socket to non-blocking + if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) //don't do it before subprocess fork! + error_exit(MSG_START "cannot set_nonblocking()"); + + for(;;) + { + //Let's wait until there is any new data to read, or any new connection! + select(highfd, &select_fds, NULL, NULL, NULL); + + //Is there a new client connection? + if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) + { + this_client = new client_t; + this_client->error = 0; + memcpy(&this_client->addr, &addr_cli, sizeof(this_client->addr)); + this_client->socket = new_socket; + if(pipe(this_client->pipefd) == -1) + { + perror(MSG_START "cannot open new pipe() for the client"); + continue; + } + if(fcntl(this_client->pipefd[1], F_SETPIPE_SZ, pipe_max_size) == -1) + perror("failed to F_SETPIPE_SZ for the client pipe"); + if(this_client->pid = fork()) + { + //We're the parent + set_nonblocking(this_client->pipefd[1]); + clients.push_back(this_client); + fprintf(stderr, MSG_START "client pid: %d\n", this_client->pid); + } + else + { + //We're the client + client(); + return 1; + } + } + + int retval = read(input_fd, buf, bufsizeall); + if(retval==0) + { + //end of input stream, close clients and exit + } + else if(retval != -1) + { + for (int i=0; ipipefd[1], buf, retval)==-1) + { + + if(!clients[i]->error) + { + print_client(clients[i], "lost buffer, failed to write pipe."); + clients[i]->error=1; + } + //fprintf(stderr, MSG_START "errno is %d\n", errno); //usually 11 + //int wpstatus; + //int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); + //fprintf(stderr, MSG_START "pid is %d\n",clients[i]->pid); + //perror("somethings wrong"); + //if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); + //else if(wpresult == 0) + waitpid(clients[i]->pid, NULL, WNOHANG); + if(!proc_exists(clients[i]->pid)) + { + //Client exited! + print_client(clients[i], "closing client from main process."); + close(clients[i]->pipefd[1]); + close(clients[i]->socket); + delete clients[i]; + clients.erase(clients.begin()+i); + fprintf(stderr, MSG_START "done closing client from main process.\n"); + } + } + else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } + } + } + //TODO: at the end, server closes pipefd[1] for client + } + + return 0; +} + +pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out, pid_t* pgrp) +{ + /*char sem_name[101]; + snprintf(sem_name,100,"ddcd_sem_%d",getpid()); + sem_t mysem; + if(sem_init(&mysem, 1, 1)==-1) error_exit("failed to sem_init() in run_subprocess()"); + fprintf(stderr, "sem_waiting\n"); + if(sem_wait(&mysem)==-1) error_exit("the first sem_wait() failed in run_subprocess()"); + fprintf(stderr, "sem_waited\n"); + */ + int syncpipe[2]; + if(pipe(syncpipe)==-1) error_exit("failed to create pipe()"); + pid_t pid = fork(); + + if(pid < 0) return 0; //fork failed + if(pid == 0) + { + setpgrp(); + write(syncpipe[1], " ", 1); + //if(sem_post(&mysem)==-1) error_exit("failed to sem_post() in run_subprocess()"); + //We're the subprocess + //fprintf(stderr, "run_subprocess :: execl\n"); + //if(fcntl(pipe_in[1], F_SETPIPE_SZ, pipe_max_size) == -1) perror("Failed to F_SETPIPE_SZ in run_subprocess()"); + if(pipe_in) + { + close(pipe_in[1]); + dup2(pipe_in[0], STDIN_FILENO); + } + if(pipe_out) + { + close(pipe_out[0]); + dup2(pipe_out[1], STDOUT_FILENO); + } + execl("/bin/bash","bash","-c",cmd, (char*)0); + error_exit(MSG_START "run_subprocess failed to execute command"); + } + else + { + //if(sem_wait(&mysem)==-1) error_exit("the second sem_wait() failed in run_subprocess()"); + int synctemp; + read(syncpipe[0], &synctemp, 1); + *pgrp = getpgid(pid); + fprintf(stderr, MSG_START "run_subprocess pgid returned = %d\n", *pgrp); + return pid; + } +} + +void print_client(client_t* client, const char* what) +{ + fprintf(stderr,MSG_START "(client %s:%d) %s\n", inet_ntoa(client->addr.sin_addr), client->addr.sin_port, what); +} + +#define CTL_BUFSIZE 1024 + +int read_socket_ctl(int fd, char* output, int max_size) +{ + //fprintf(stderr, "doing read_socket_ctl %d\n", fd); + //if(!fd) return 0; + static char buffer[CTL_BUFSIZE]; + static int buffer_index=0; + if(buffer_index==CTL_BUFSIZE) buffer_index=0; + int bytes_read=recv(fd,buffer+buffer_index,(CTL_BUFSIZE-buffer_index)*sizeof(char), MSG_DONTWAIT); + if(bytes_read<=0) return 0; + //fprintf(stderr, "recv %d\n", bytes_read); + + int prev_newline_at=0; + int last_newline_at=0; + for(int i=0;ipipefd[0]; + int pipe_ctl[2], pipe_stdout[2]; + + prctl(PR_SET_PDEATHSIG, SIGHUP); //get a signal when parent exits + + if(decimation!=1) + { + + if(pipe(pipe_ctl)==-1) error_exit(MSG_START "cannot open new pipe() for the client subprocess"); + if(pipe(pipe_stdout)==-1) error_exit(MSG_START "cannot open new pipe() for the client subprocess"); + switch(ddc_method) + { + case M_TD: + sprintf(client_subprocess_cmd_buf, subprocess_cmd_td, pipe_ctl[0], decimation, transition_bw); + break; + case M_FASTDDC: + sprintf(client_subprocess_cmd_buf, subprocess_args_fastddc_2, pipe_ctl[0], decimation, transition_bw); + break; + } + + if(!(client_subprocess_pid = run_subprocess( client_subprocess_cmd_buf, this_client->pipefd, pipe_stdout, &client_subprocess_pgrp))) + print_exit(MSG_START "couldn't start client_subprocess_cmd!\n"); + fprintf(stderr, MSG_START "starting client_subprocess_cmd: %s\n", client_subprocess_cmd_buf); + input_fd = pipe_stdout[0]; //we don't have to set it nonblocking + fprintf(stderr, MSG_START "pipe_stdout[0] = %d\n", pipe_stdout[0]); + write(pipe_ctl[1], "0.0\n", 4); + } + char recv_cmd[CTL_BUFSIZE]; + char temps[CTL_BUFSIZE*2]; + int tempi; + float tempf; + + for(;;) + { + while(read_socket_ctl(this_client->socket, recv_cmd, CTL_BUFSIZE)) + { + sprintf(temps, "read_socket_ctl: %s", recv_cmd); + print_client(this_client, temps); + if(ctl_get_arg(recv_cmd, "bypass", "%d", &tempi)) + { + if(tempi==1 && client_subprocess_pid) + { + //print_client(this_client, "suspending client_subprocess_pgrp...\n"); + //fprintf(stderr, "client_subprocess_pgrp = %d\n", client_subprocess_pgrp); + //killpg(client_subprocess_pgrp, SIGTSTP); + //while(proc_exists(client_subprocess_pid)) usleep(10000); + //print_client(this_client, "done killing client_subprocess_pid.\n"); + input_fd=this_client->pipefd[0]; //by doing this, we don't read from pipe_stdout[0] anymore, so that csdr stops doing anything, and also doesn't read anymore from the input: we get the whole I/Q stream! + } + if(tempi==0 && client_subprocess_pid) + { + input_fd=pipe_stdout[0]; + } + + } + if(ctl_get_arg(recv_cmd, "shift", "%g", &tempf)) + { + tempi=sprintf(temps, "%g\n", tempf); + write(pipe_ctl[1], temps, tempi); + fsync(pipe_ctl[1]); + } + } + int nread = read(input_fd,buf,bufsizeall); + if(nread<=0) continue; + if(send(this_client->socket,buf,nread,0)==-1) + { + print_client(this_client, "client process is exiting.\n"); + if(client_subprocess_pid) killpg2(client_subprocess_pgrp); + exit(0); + } + } +} + +void killpg2(pid_t pgrp) +{ + //fprintf(stderr, MSG_START "killpg2: %d\n", pgrp); + if(pgrp!=1 && pgrp!=0) killpg(pgrp, SIGTERM); +} + +void error_exit(const char* why) +{ + perror(why); + exit(1); +} + +void print_exit(const char* why) +{ + fprintf(stderr, "%s", why); + exit(1); +} + +void maxfd(int* maxfd, int fd) +{ + if(fd>=*maxfd) *maxfd=fd+1; +} diff --git a/ddcd_old.h b/ddcd_old.h new file mode 100644 index 0000000..af4cfb5 --- /dev/null +++ b/ddcd_old.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct client_s +{ + struct sockaddr_in addr; + int socket; + pid_t pid; + int pipefd[2]; + int error; + pid_t dsp_proc; +} client_t; + + +void client(); +void error_exit(const char* why); +void print_exit(const char* why); +void print_client(client_t* client, const char* what); +int proc_exists(pid_t pid); +pid_t run_subprocess(char* cmd, int* pipe_in, int* pipe_out, pid_t* pgrp); +void maxfd(int* maxfd, int fd); +void sig_handler(int signo); +void killpg2(pid_t pgrp); +int ctl_get_arg(char* input, const char* cmd, const char* format, ...); + +typedef enum ddc_method_e +{ + M_TD, + M_FASTDDC +} ddc_method_t; + +const char subprocess_cmd_td[] = "csdr " +#ifdef NEON_OPTS + "shift_addfast_cc" +#else + "shift_unroll_cc" +#endif + " --fd %d | csdr fir_decimate_cc %d %g"; + +const char subprocess_args_fastddc_1[] = "csdr fastddc_fwd_cc %d %g"; +//const char subprocess_args_fastddc_1[] = "csdr through %d %g"; +const char subprocess_args_fastddc_2[] = "csdr fastddc_inv_cc --fd %d %d %g"; +//const char subprocess_args_fastddc_2[] = "csdr convert_u8_f %d %d %g"; From fc19371a3e54f9ad08a08f287773a6fa345e2863 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 12 Jan 2016 23:19:35 +0100 Subject: [PATCH 34/74] Documentation++ --- README.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1742454..6aabad9 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ Internally, a sine and cosine wave is generated to perform this function, and th shift_addition_cc -Operation is the same as with `shift_math_cc`. +Operation is the same as for `shift_math_cc`. Internally, this function uses trigonometric addition formulas to generate sine and cosine, which is a bit faster. (About 4 times on the machine I have tested it on.) @@ -203,6 +203,20 @@ Operation is the same as with `shift_math_cc`. Internally, this function uses a look-up table (LUT) to recall the values of the sine function (for the first quadrant). The higher the table size is, the smaller the phase error is. + shift_addfast_cc + +Operation is the same as for `shift_math_cc`. + +Internally, this function uses a NEON-accelerated algorithm on capable systems, so it is advised to use this one on ARM boards. + + shift_unroll_cc + +Operation is the same as for `shift_math_cc`. + +This uses a modified algoritm that first stores a vector of sine and cosine values for given phase differences. + +The loop in this function unrolls quite well if compiled on a PC. It was the fastest one on an i7 CPU during the tests. + decimating_shift_addition_cc [decimation] It shifts the input signal in the frequency domain, and also decimates it, without filtering. It will be useful as a part of the FFT channelizer implementation (to be done). @@ -347,6 +361,12 @@ The actual number of padding samples can be determined by running `cat csdr.c | It exchanges the first and second part of the FFT vector, to prepare it for the waterfall/spectrum display. It should operate on the data output from `logpower_cf`. + setbuf + +If the environment variable `CSDR_DYNAMIC_BUFSIZE_ON` is set to 1, then you can use this command to set the input buffer size for the next `csdr` process in the chain. + + + #### Control via pipes Some parameters can be changed while the `csdr` process is running. To achieve this, some `csdr` functions have special parameters. You have to supply a fifo previously created by the `mkfifo` command. Processing will only start after the first control command has been received by `csdr` over the FIFO. From 4737a7e808b180d6f67b04a9a8ba30432cf38246 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sat, 4 Jun 2016 17:42:29 +0200 Subject: [PATCH 35/74] Current state --- ddcd.cpp | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++++ ddcd.h | 13 ++++- tsmpool.cpp | 53 +++++++++++++++++++ tsmpool.h | 29 +++++++++++ 4 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 tsmpool.cpp create mode 100644 tsmpool.h diff --git a/ddcd.cpp b/ddcd.cpp index 5222022..f8fe9d6 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -35,9 +35,19 @@ char host_address[100] = "127.0.0.1"; int decimation = 0; float transition_bw = 0.05; int bufsize = 1024; +int bufcnt = 1024; +int maxbufcnt = 1000 +int thread_cntr = 0; char ddc_method_str[100] = "td"; ddc_method_t ddc_method; +void sig_handler(int signo) +{ + fprintf(stderr, MSG_START "signal %d caught, exiting ddcd...\n", signo); + fflush(stderr); + exit(0); +} + int main(int argc, char* argv[]) { int c; @@ -103,7 +113,135 @@ int main(int argc, char* argv[]) } else print_exit(MSG_START "invalid parameter given to --method.\n"); + //set signals + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sig_handler; + sigaction(SIGTERM, &sa, NULL); + sigaction(SIGKILL, &sa, NULL); + sigaction(SIGQUIT, &sa, NULL); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGHUP, &sa, NULL); + + struct sockaddr_in addr_host; + int listen_socket; + std::vector clients; + clients.reserve(100); + listen_socket=socket(AF_INET,SOCK_STREAM,0); + int sockopt = 1; + if( setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&sockopt, sizeof(sockopt)) == -1 ) + error_exit(MSG_START "cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 + + memset(&addr_host,'0',sizeof(addr_host)); + addr_host.sin_family = AF_INET; + addr_host.sin_port = htons(host_port); + addr_host.sin_addr.s_addr = INADDR_ANY; + + if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) + error_exit(MSG_START "invalid host address"); + + if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) + error_exit(MSG_START "cannot bind() address to the socket"); + + if( listen(listen_socket, 10) == -1 ) + error_exit(MSG_START "cannot listen() on socket"); + + fprintf(stderr,MSG_START "listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); + + struct sockaddr_in addr_cli; + socklen_t addr_cli_len = sizeof(addr_cli); + int new_socket; + + int highfd = 0; + FD_ZERO(&select_fds); + FD_SET(listen_socket, &select_fds); + maxfd(&highfd, listen_socket); + FD_SET(input_fd, &select_fds); + maxfd(&highfd, input_fd); + + //Set stdin and listen_socket to non-blocking + if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) //don't do it before subprocess fork! + error_exit(MSG_START "cannot set_nonblocking()"); + + for(;;) + { + //Let's wait until there is any new data to read, or any new connection! + select(highfd, &select_fds, NULL, NULL, NULL); + + //Is there a new client connection? + if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) + { + this_client = new client_t; + this_client->error = 0; + memcpy(&this_client->addr, &addr_cli, sizeof(this_client->addr)); + this_client->socket = new_socket; + this_client->id = thread_cntr++; + + if(pthread_create(&this_client->thread, NULL, client_thread , (void*)&this_client)<0) + { + //We're the parent + clients.push_back(this_client); + fprintf(stderr, MSG_START "pthread_create() done, this_client->id: %d\n", this_client->id); + } + } + + float* pool_next = pool->get_write_buffer(); + + int retval = read(input_fd, pool_next, mainpool->size); + if(retval==0) + { + //end of input stream, close clients and exit + } + else if(retval != -1) + { + for (int i=0; ipipefd[1], buf, retval)==-1) + { + + if(!clients[i]->error) + { + print_client(clients[i], "lost buffer, failed to write pipe."); + clients[i]->error=1; + } + //fprintf(stderr, MSG_START "errno is %d\n", errno); //usually 11 + //int wpstatus; + //int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); + //fprintf(stderr, MSG_START "pid is %d\n",clients[i]->pid); + //perror("somethings wrong"); + //if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); + //else if(wpresult == 0) + waitpid(clients[i]->pid, NULL, WNOHANG); + if(!proc_exists(clients[i]->pid)) + { + //Client exited! + print_client(clients[i], "closing client from main process."); + close(clients[i]->pipefd[1]); + close(clients[i]->socket); + delete clients[i]; + clients.erase(clients.begin()+i); + fprintf(stderr, MSG_START "done closing client from main process.\n"); + } + } + else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } + } + } + //TODO: at the end, server closes pipefd[1] for client + + } + + +} + +void* client_thread (void* param) +{ +} + +void error_exit(const char* why) +{ + perror(why); + exit(1); } void print_exit(const char* why) @@ -111,3 +249,9 @@ void print_exit(const char* why) fprintf(stderr, "%s", why); exit(1); } + +void maxfd(int* maxfd, int fd) +{ + if(fd>=*maxfd) *maxfd=fd+1; +} + diff --git a/ddcd.h b/ddcd.h index 845cc33..5436476 100644 --- a/ddcd.h +++ b/ddcd.h @@ -4,6 +4,11 @@ #include #include #include +#include +#include +#include +#include +#include #define SOFTWARE_NAME "ddcd" #define MSG_START SOFTWARE_NAME ": " @@ -14,13 +19,17 @@ typedef enum ddc_method_e M_FASTDDC } ddc_method_t; -void print_exit(const char* why); - typedef struct client_s { struct sockaddr_in addr; + int id; int socket; int error; pthread_t thread; } client_t; +void print_exit(const char* why); +void error_exit(const char* why); +void maxfd(int* maxfd, int fd); + + diff --git a/tsmpool.cpp b/tsmpool.cpp new file mode 100644 index 0000000..3af0149 --- /dev/null +++ b/tsmpool.cpp @@ -0,0 +1,53 @@ +tsmpool::tsmpool(size_t size, int num) +{ + this->threads_cntr = 0; + this->num = num; + this->size = size; + this->ok = 1; + this->lowest_read_index = -1; + if (pthread_mutex_init(&this->mutex, NULL) != 0) this->ok=0; +} + +size_t tsmpool::get_size() { return this->size; } + +void* tsmpool::get_write_buffer() +{ + if(write_index==index_before(lowest_read_index)) return NULL; + void* to_return = buffers[write_index]; + write_index=index_next(write_index); +} + +tsmthread_t* tsmpool::register_thread() +{ + if(!ok) return -1; + pthread_mutex_lock(&this->mutex); + tsmthread_t* thread = new tsmthread_t; + thread->read_index = write_index; + threads.push_back(thread); + pthread_mutex_unlock(&this->mutex); + return thread; +} + +int tsmpool::remove_thread(tsmthread_t* thread) +{ + pthread_mutex_lock(&this->mutex); + for(int i=0;imutex); +} + +void* tsmpool::get_read_buffer(tsmthread_t* thread) +{ + if(thread->read_index==write_index) return NULL; + void* to_return = buffers[thread->read_index]; + thread->read_index=index_next(thread->read_index); +} + +void* tsmpool::set_read_index_distance(tsmthread_t* thread, int distance) +{ +} diff --git a/tsmpool.h b/tsmpool.h new file mode 100644 index 0000000..aa8385c --- /dev/null +++ b/tsmpool.h @@ -0,0 +1,29 @@ +typedef struct tsmthread_s +{ + int read_index; //it always points to the next buffer to be read +} tsmthread_t; + +class tsmpool +{ +private: + size_t size; + int num; + vector threads; + vector buffers; + int threads_cntr; + pthread_mutex_t mutex; + int ok; + int write_index; //it always points to the next buffer to be written + int lowest_read_index; + +public: + size_t get_size(); + tsmpool(size_t size, int num); + void* get_write_buffer(); + int register_thread(); + void* get_read_buffer(int thread_id); + int index_next(int index) { return (index+1==size)?0:index; } + int index_before(int index) { return (index-1<0)?size-1:index; } +} + + From 461390edf75763be5e54d44ebcfd636a117adea8 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sat, 4 Jun 2016 22:54:17 +0200 Subject: [PATCH 36/74] Written some more code for ddcd --- ddcd.cpp | 197 ++++++++++++++++++++++++++++++++-------------------- ddcd.h | 17 +++-- tsmpool.cpp | 12 ++-- tsmpool.h | 13 ++-- 4 files changed, 151 insertions(+), 88 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index f8fe9d6..c62abe5 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -1,5 +1,5 @@ /* -This software is part of libcsdr, a set of simple DSP routines for +This software is part of libcsdr, a set of simple DSP routines for Software Defined Radio. Copyright (c) 2014, Andras Retzler @@ -30,19 +30,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "ddcd.h" + int host_port = 0; char host_address[100] = "127.0.0.1"; +int thread_cntr = 0; + +//CLI parameters int decimation = 0; float transition_bw = 0.05; -int bufsize = 1024; +int bufsize = 1024; //! currently unused int bufcnt = 1024; -int maxbufcnt = 1000 -int thread_cntr = 0; char ddc_method_str[100] = "td"; ddc_method_t ddc_method; void sig_handler(int signo) -{ +{ fprintf(stderr, MSG_START "signal %d caught, exiting ddcd...\n", signo); fflush(stderr); exit(0); @@ -59,12 +61,13 @@ int main(int argc, char* argv[]) {"address", required_argument, 0, 'a' }, {"decimation", required_argument, 0, 'd' }, {"bufsize", required_argument, 0, 'b' }, + {"bufcnt", required_argument, 0, 'n' }, {"method", required_argument, 0, 'm' }, {"transition", required_argument, 0, 't' } }; - c = getopt_long(argc, argv, "p:a:d:b:m:t:", long_options, &option_index); + c = getopt_long(argc, argv, "p:a:d:b:n:m:t:", long_options, &option_index); if(c==-1) break; - switch (c) + switch (c) { case 'a': host_address[100-1]=0; @@ -79,6 +82,9 @@ int main(int argc, char* argv[]) case 'b': bufsize=atoi(optarg); break; + case 'n': + bufcnt=atoi(optarg); + break; case 'm': ddc_method_str[100-1]=0; strncpy(ddc_method_str,optarg,100-1); @@ -93,22 +99,23 @@ int main(int argc, char* argv[]) print_exit(MSG_START "error in getopt_long()\n"); } } - + if(!decimation) print_exit(MSG_START "missing required command line argument, --decimation.\n"); if(!host_port) print_exit(MSG_START "missing required command line argument, --port.\n"); if(decimation<0) print_exit(MSG_START "invalid value for --decimation (should be >0).\n"); if(decimation==1) fprintf(stderr, MSG_START "decimation = 1, just copying raw samples.\n"); if(transition_bw<0||transition_bw>0.5) print_exit(MSG_START "invalid value for --transition (should be between 0 and 0.5).\n"); - - if(decimation==1); //don't do anything then - else if(!strcmp(ddc_method_str,"td")) + if(bufsize<0) print_exit(MSG_START "invalid value for --bufsize (should be >0)\n"); + if(bufcnt<0) print_exit(MSG_START "invalid value for --bufcnt (should be >0)\n"); + if(decimation==1); //don't do anything then //!will have to take care about this later + else if(!strcmp(ddc_method_str,"td")) { - ddc_method = M_TD; + ddc_method = M_TD; fprintf(stderr, MSG_START "method is M_TD (default).\n"); } - else if (!strcmp(ddc_method_str,"fastddc")) + else if (!strcmp(ddc_method_str,"fastddc")) { - ddc_method = M_FASTDDC; + ddc_method = M_FASTDDC; fprintf(stderr, MSG_START "method is M_FASTDDC.\n"); } else print_exit(MSG_START "invalid parameter given to --method.\n"); @@ -122,7 +129,7 @@ int main(int argc, char* argv[]) sigaction(SIGQUIT, &sa, NULL); sigaction(SIGINT, &sa, NULL); sigaction(SIGHUP, &sa, NULL); - + struct sockaddr_in addr_host; int listen_socket; std::vector clients; @@ -132,13 +139,13 @@ int main(int argc, char* argv[]) int sockopt = 1; if( setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&sockopt, sizeof(sockopt)) == -1 ) error_exit(MSG_START "cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 - + memset(&addr_host,'0',sizeof(addr_host)); addr_host.sin_family = AF_INET; addr_host.sin_port = htons(host_port); addr_host.sin_addr.s_addr = INADDR_ANY; - if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) + if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) error_exit(MSG_START "invalid host address"); if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) @@ -153,17 +160,24 @@ int main(int argc, char* argv[]) socklen_t addr_cli_len = sizeof(addr_cli); int new_socket; - int highfd = 0; + int highfd = 0; FD_ZERO(&select_fds); FD_SET(listen_socket, &select_fds); maxfd(&highfd, listen_socket); FD_SET(input_fd, &select_fds); maxfd(&highfd, input_fd); - //Set stdin and listen_socket to non-blocking - if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) //don't do it before subprocess fork! + //Set stdin and listen_socket to non-blocking + if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) error_exit(MSG_START "cannot set_nonblocking()"); + //Create tsmpool + tsmpool* pool = new tsmpool(bufsize, bufcnt); + + unsigned char* current_write_buffer = pool->get_write_buffer(); + int index_in_current_write_buffer = 0; + + for(;;) { //Let's wait until there is any new data to read, or any new connection! @@ -171,71 +185,107 @@ int main(int argc, char* argv[]) //Is there a new client connection? if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) - { - this_client = new client_t; - this_client->error = 0; - memcpy(&this_client->addr, &addr_cli, sizeof(this_client->addr)); - this_client->socket = new_socket; - this_client->id = thread_cntr++; - - if(pthread_create(&this_client->thread, NULL, client_thread , (void*)&this_client)<0) + { + clients_close_all_finished(); + int new_client_id = clients_get_new_id(); + if(new_client_id!=-1 && pthread_create(&new_client->thread, NULL, client_thread , (void*)&new_client)<0) { //We're the parent - clients.push_back(this_client); - fprintf(stderr, MSG_START "pthread_create() done, this_client->id: %d\n", this_client->id); + client_t* new_client = new client_t; + new_client->error = 0; + memcpy(&new_client->addr, &addr_cli, sizeof(new_client->addr)); + new_client->socket = new_socket; + new_client->id = new_client_id; + new_client->status = CS_CREATED; + clients.push_back(new_client); + fprintf(stderr, MSG_START "pthread_create() done, new_client->id: %d\n", new_client->id); } + else fprintf(stderr, MSG_START "pthread_create() failed."); } - float* pool_next = pool->get_write_buffer(); - - int retval = read(input_fd, pool_next, mainpool->size); - if(retval==0) + if(index_in_current_write_buffer >= bufsize) { - //end of input stream, close clients and exit + current_write_buffer = pool->get_write_buffer(); + index_in_current_write_buffer = 0; } - else if(retval != -1) + int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); + if(retval>0) { - for (int i=0; ipipefd[1], buf, retval)==-1) - { - - if(!clients[i]->error) - { - print_client(clients[i], "lost buffer, failed to write pipe."); - clients[i]->error=1; - } - //fprintf(stderr, MSG_START "errno is %d\n", errno); //usually 11 - //int wpstatus; - //int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); - //fprintf(stderr, MSG_START "pid is %d\n",clients[i]->pid); - //perror("somethings wrong"); - //if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); - //else if(wpresult == 0) - waitpid(clients[i]->pid, NULL, WNOHANG); - if(!proc_exists(clients[i]->pid)) - { - //Client exited! - print_client(clients[i], "closing client from main process."); - close(clients[i]->pipefd[1]); - close(clients[i]->socket); - delete clients[i]; - clients.erase(clients.begin()+i); - fprintf(stderr, MSG_START "done closing client from main process.\n"); - } - } - else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } - } + index_in_current_write_buffer += retval; + } + else if(retval==0) + { + //!end of input stream, close clients and exit + print_exit(MSG_START "end of input, exiting.\n") } - //TODO: at the end, server closes pipefd[1] for client - } - - } -void* client_thread (void* param) +#if 0 +for (int i=0; ipipefd[1], buf, retval)==-1) + { + + if(!clients[i]->error) + { + print_client(clients[i], "lost buffer, failed to write pipe."); + clients[i]->error=1; + } + //fprintf(stderr, MSG_START "errno is %d\n", errno); //usually 11 + //int wpstatus; + //int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); + //fprintf(stderr, MSG_START "pid is %d\n",clients[i]->pid); + //perror("somethings wrong"); + //if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); + //else if(wpresult == 0) + waitpid(clients[i]->pid, NULL, WNOHANG); + if(!proc_exists(clients[i]->pid)) + { + //Client exited! + print_client(clients[i], "closing client from main process."); + close(clients[i]->pipefd[1]); + close(clients[i]->socket); + delete clients[i]; + clients.erase(clients.begin()+i); + fprintf(stderr, MSG_START "done closing client from main process.\n"); + } + } + else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } +} +} +//TODO: at the end, server closes pipefd[1] for client +#endif + +int clients_get_new_id() +{ + int new_id=-1; + for(int i=0; iid) new_id = clients[i]->id; + if(new_id!=INT_MAX) return ++new_id; //will also work if clients is empty (will return -1+1==0) + //should test this part, too: + for(new_id=0;new_idid==new_id) { found = 1; break; } + if(found) continue; + else return new_id; + } + return -1; +} + +void clients_close_all_finished() +{ + for(int i=0;istatus == CS_THREAD_FINISHED) clients.erase(i); + } +} + +void* client_thread (void* param) //!TODO +{ + client_t* me_the_client = (client_t*)param; + + return NULL; } void error_exit(const char* why) @@ -252,6 +302,5 @@ void print_exit(const char* why) void maxfd(int* maxfd, int fd) { - if(fd>=*maxfd) *maxfd=fd+1; + if(fd>=*maxfd) *maxfd=fd+1; } - diff --git a/ddcd.h b/ddcd.h index 5436476..a6f67b1 100644 --- a/ddcd.h +++ b/ddcd.h @@ -9,27 +9,36 @@ #include #include #include +#include #define SOFTWARE_NAME "ddcd" #define MSG_START SOFTWARE_NAME ": " -typedef enum ddc_method_e +typedef enum ddc_method_e { M_TD, M_FASTDDC } ddc_method_t; +typedef enum client_status_e +{ + CS_CREATED, + CS_THREAD_RUNNING, + CS_THREAD_FINISHED +} client_status_t; + + typedef struct client_s { struct sockaddr_in addr; int id; int socket; - int error; + int error; //set to non-zero on error (data transfer failed) pthread_t thread; + client_status_t status; + } client_t; void print_exit(const char* why); void error_exit(const char* why); void maxfd(int* maxfd, int fd); - - diff --git a/tsmpool.cpp b/tsmpool.cpp index 3af0149..237ef44 100644 --- a/tsmpool.cpp +++ b/tsmpool.cpp @@ -1,8 +1,10 @@ -tsmpool::tsmpool(size_t size, int num) +#include "tsmpool.h" + +tsmpool::tsmpool(size_t size, int num) { this->threads_cntr = 0; - this->num = num; this->size = size; + this->num = num; //number of buffers of (size) to alloc this->ok = 1; this->lowest_read_index = -1; if (pthread_mutex_init(&this->mutex, NULL) != 0) this->ok=0; @@ -10,8 +12,8 @@ tsmpool::tsmpool(size_t size, int num) size_t tsmpool::get_size() { return this->size; } -void* tsmpool::get_write_buffer() -{ +void* tsmpool::get_write_buffer() +{ if(write_index==index_before(lowest_read_index)) return NULL; void* to_return = buffers[write_index]; write_index=index_next(write_index); @@ -19,7 +21,7 @@ void* tsmpool::get_write_buffer() tsmthread_t* tsmpool::register_thread() { - if(!ok) return -1; + if(!ok) return NULL; pthread_mutex_lock(&this->mutex); tsmthread_t* thread = new tsmthread_t; thread->read_index = write_index; diff --git a/tsmpool.h b/tsmpool.h index aa8385c..f855226 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -1,7 +1,12 @@ -typedef struct tsmthread_s -{ +//tsmpool stands for Thread-Safe Memory Pool. + +//It implements a big circular buffer that one thread writes into, and multiple threads read from. +//The reader threads have lower priority than the writer thread (they can be left behind if the don't read fast enough). + +typedef struct tsmthread_s +{ int read_index; //it always points to the next buffer to be read -} tsmthread_t; +} tsmthread_t; class tsmpool { @@ -25,5 +30,3 @@ public: int index_next(int index) { return (index+1==size)?0:index; } int index_before(int index) { return (index-1<0)?size-1:index; } } - - From 3a38be042a3000d78ad63c0b9d11199a943c2e93 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sat, 4 Jun 2016 22:56:20 +0200 Subject: [PATCH 37/74] We don't need the client id at all. --- ddcd.cpp | 22 ++-------------------- ddcd.h | 1 - 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index c62abe5..ef428b7 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -187,18 +187,16 @@ int main(int argc, char* argv[]) if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) { clients_close_all_finished(); - int new_client_id = clients_get_new_id(); - if(new_client_id!=-1 && pthread_create(&new_client->thread, NULL, client_thread , (void*)&new_client)<0) + if(pthread_create(&new_client->thread, NULL, client_thread , (void*)&new_client)<0) { //We're the parent client_t* new_client = new client_t; new_client->error = 0; memcpy(&new_client->addr, &addr_cli, sizeof(new_client->addr)); new_client->socket = new_socket; - new_client->id = new_client_id; new_client->status = CS_CREATED; clients.push_back(new_client); - fprintf(stderr, MSG_START "pthread_create() done, new_client->id: %d\n", new_client->id); + fprintf(stderr, MSG_START "pthread_create() done, clients now: %d\n", clients.size()); } else fprintf(stderr, MSG_START "pthread_create() failed."); } @@ -257,22 +255,6 @@ for (int i=0; iid) new_id = clients[i]->id; - if(new_id!=INT_MAX) return ++new_id; //will also work if clients is empty (will return -1+1==0) - //should test this part, too: - for(new_id=0;new_idid==new_id) { found = 1; break; } - if(found) continue; - else return new_id; - } - return -1; -} - void clients_close_all_finished() { for(int i=0;i Date: Sat, 4 Jun 2016 23:05:38 +0200 Subject: [PATCH 38/74] Don't allow readers reach the buffer currently being filled. --- ddcd.cpp | 2 ++ tsmpool.cpp | 7 ++++--- tsmpool.h | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index ef428b7..e16a5c2 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -267,6 +267,8 @@ void* client_thread (void* param) //!TODO { client_t* me_the_client = (client_t*)param; + + pthread_exit(NULL); return NULL; } diff --git a/tsmpool.cpp b/tsmpool.cpp index 237ef44..6593595 100644 --- a/tsmpool.cpp +++ b/tsmpool.cpp @@ -7,6 +7,7 @@ tsmpool::tsmpool(size_t size, int num) this->num = num; //number of buffers of (size) to alloc this->ok = 1; this->lowest_read_index = -1; + this->write_index = 0; if (pthread_mutex_init(&this->mutex, NULL) != 0) this->ok=0; } @@ -14,7 +15,7 @@ size_t tsmpool::get_size() { return this->size; } void* tsmpool::get_write_buffer() { - if(write_index==index_before(lowest_read_index)) return NULL; + //if(write_index==index_before(lowest_read_index)) return NULL; void* to_return = buffers[write_index]; write_index=index_next(write_index); } @@ -24,7 +25,7 @@ tsmthread_t* tsmpool::register_thread() if(!ok) return NULL; pthread_mutex_lock(&this->mutex); tsmthread_t* thread = new tsmthread_t; - thread->read_index = write_index; + thread->read_index = index_before(write_index); threads.push_back(thread); pthread_mutex_unlock(&this->mutex); return thread; @@ -45,7 +46,7 @@ int tsmpool::remove_thread(tsmthread_t* thread) void* tsmpool::get_read_buffer(tsmthread_t* thread) { - if(thread->read_index==write_index) return NULL; + if(thread->read_index==index_before(write_index)) return NULL; void* to_return = buffers[thread->read_index]; thread->read_index=index_next(thread->read_index); } diff --git a/tsmpool.h b/tsmpool.h index f855226..2f097d1 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -19,7 +19,7 @@ private: pthread_mutex_t mutex; int ok; int write_index; //it always points to the next buffer to be written - int lowest_read_index; + int lowest_read_index; //unused public: size_t get_size(); From f9d6d22fe2d3d77a1129d9fa2ee293417351354f Mon Sep 17 00:00:00 2001 From: Tatu Peltola Date: Fri, 25 Mar 2016 18:00:20 +0200 Subject: [PATCH 39/74] Implement averaged FFT --- csdr.c | 38 ++++++++++++++++++++++++++++++++++++++ libcsdr.c | 13 +++++++++++++ libcsdr.h | 2 ++ 3 files changed, 53 insertions(+) diff --git a/csdr.c b/csdr.c index 52de1ab..1ed5478 100644 --- a/csdr.c +++ b/csdr.c @@ -1299,6 +1299,44 @@ int main(int argc, char *argv[]) } } + if(!strcmp(argv[1],"logaveragepower_cf")) + { + bigbufs=1; + if(argc<=4) return badsyntax("need required parameters (add_db, table_size, avgnumber)"); + float add_db=0; + int avgnumber=0; + int fft_size=0; + + sscanf(argv[2],"%g",&add_db); + sscanf(argv[3],"%d",&fft_size); + sscanf(argv[4],"%d",&avgnumber); + + if(!getbufsize()) return -2; //dummy + if(!sendbufsize(initialize_buffers())) return -2; + + if(fft_size != the_bufsize) return -2; + + //fprintf(stderr, "logaveragepower_cf %f %d=%d %d\n", add_db, fft_size, the_bufsize, avgnumber); + add_db -= 10*log10(avgnumber); + for(;;) + { + int i,n; + for(i = 0; i < the_bufsize; i++) { + output_buffer[i] = 0; + } + FEOF_CHECK; + for(n = 0; n < avgnumber; n++) { + FREAD_C; + //fprintf(stderr, "averaged %d\n", n); + accumulate_power_cf((complexf*)input_buffer, output_buffer, the_bufsize); + } + log_ff(NULL, output_buffer, the_bufsize, add_db); + FWRITE_R; + TRY_YIELD; + } + return 0; + } + if(!strcmp(argv[1],"fft_exchange_sides_ff")) { if(argc<=2) return badsyntax("need required parameters (fft_size)"); diff --git a/libcsdr.c b/libcsdr.c index e3b01bf..0be4cd0 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -949,6 +949,19 @@ void logpower_cf(complexf* input, float* output, int size, float add_db) for(int i=0;i Date: Sat, 26 Mar 2016 11:44:28 +0200 Subject: [PATCH 40/74] Precalculate FFT window Before this calculating FFT window was taking much more CPU time than calculating the FFT itself. --- csdr.c | 5 ++++- libcsdr.c | 23 +++++++++++++++++++++++ libcsdr.h | 2 ++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/csdr.c b/csdr.c index 1ed5478..c8e8d90 100644 --- a/csdr.c +++ b/csdr.c @@ -1245,6 +1245,8 @@ int main(int argc, char *argv[]) FFT_PLAN_T* plan=make_fft_c2c(fft_size, windowed, output, 1, benchmark); if(benchmark) fprintf(stderr," done\n"); if(octave) printf("setenv(\"GNUTERM\",\"X11 noraise\");y=zeros(1,%d);semilogy(y,\"ydatasource\",\"y\");\n",fft_size); + float *windowt; + windowt = precalculate_window(fft_size, window); for(;;) { FEOF_CHECK; @@ -1263,7 +1265,8 @@ int main(int argc, char *argv[]) for(int i=0;i Date: Tue, 7 Jun 2016 22:14:36 +0200 Subject: [PATCH 41/74] Added a lot of things, tsmpool should now work in 1R1W mode as well --- ddcd.cpp | 43 ++++++++++++++++++++++++++++++++++++++++--- ddcd.h | 13 +++++++++++++ tsmpool.cpp | 7 ++++--- tsmpool.h | 1 + 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index e16a5c2..9a76d56 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -173,6 +173,7 @@ int main(int argc, char* argv[]) //Create tsmpool tsmpool* pool = new tsmpool(bufsize, bufcnt); + if(!pool->ok) print_exit(MSG_START "tsmpool failed to initialize\n"); unsigned char* current_write_buffer = pool->get_write_buffer(); int index_in_current_write_buffer = 0; @@ -198,13 +199,13 @@ int main(int argc, char* argv[]) clients.push_back(new_client); fprintf(stderr, MSG_START "pthread_create() done, clients now: %d\n", clients.size()); } - else fprintf(stderr, MSG_START "pthread_create() failed."); + else fprintf(stderr, MSG_START "pthread_create() failed.\n"); } if(index_in_current_write_buffer >= bufsize) { current_write_buffer = pool->get_write_buffer(); - index_in_current_write_buffer = 0; + index_in_current_write_buffer = 0;error_exiterror_exit } int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); if(retval>0) @@ -263,18 +264,54 @@ void clients_close_all_finished() } } +void client_parser_push(char c) +{ //!TODO + command_t cmd; + char* commands_cstr = commands.c_str(); + int newline_index = -1; + + for(int i=0;commands_cstr[i];i++) if(commands_cstr[i]=='\n') newline_index = i; + if(newline_index == -1) + + char param_name[101]; + char param_value[101]; + for(int i=0;i<100;commands_csdr + +} + void* client_thread (void* param) //!TODO { client_t* me_the_client = (client_t*)param; + me_the_client->status = CS_THREAD_RUNNING; + char ctl_data_buffer; + int retval; + for(;;) + { + do + { + retval = recv(me_the_client->socket, &ctl_data_buffer, 1, 0); + if(client_parser_push(ctl_data_buffer)) break; + } while (retval); + //read control data from socket + //process control data + //run shift + //run decimation + //have an exit condition (??) + if(ddc_method == M_TD) + { + + } + } + me_the_client->status = CS_THREAD_FINISHED; pthread_exit(NULL); return NULL; } void error_exit(const char* why) { - perror(why); + perror(why); //do we need a \n at the end of (why)? exit(1); } diff --git a/ddcd.h b/ddcd.h index 37cb658..766be47 100644 --- a/ddcd.h +++ b/ddcd.h @@ -38,6 +38,19 @@ typedef struct client_s } client_t; +typedef enum command_type_e +{ + CT_SHIFT, + CT_BYPASS +} command_type_t; + + +typedef struct command_s +{ + command_type_t type; + float float_param; +} command_t; + void print_exit(const char* why); void error_exit(const char* why); void maxfd(int* maxfd, int fd); diff --git a/tsmpool.cpp b/tsmpool.cpp index 6593595..67507c5 100644 --- a/tsmpool.cpp +++ b/tsmpool.cpp @@ -46,9 +46,10 @@ int tsmpool::remove_thread(tsmthread_t* thread) void* tsmpool::get_read_buffer(tsmthread_t* thread) { - if(thread->read_index==index_before(write_index)) return NULL; - void* to_return = buffers[thread->read_index]; - thread->read_index=index_next(thread->read_index); + int* actual_read_index = (thread==NULL) ? &my_read_index : &thread->read_index; + if(*actual_read_index==index_before(write_index)) return NULL; + void* to_return = buffers[*actual_read_index]; + *actual_read_index=index_next(*actual_read_index); } void* tsmpool::set_read_index_distance(tsmthread_t* thread, int distance) diff --git a/tsmpool.h b/tsmpool.h index 2f097d1..f4901cc 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -20,6 +20,7 @@ private: int ok; int write_index; //it always points to the next buffer to be written int lowest_read_index; //unused + int my_read_index; //it is used when tsmpool is used as a single writer - single reader circular buffer public: size_t get_size(); From 91c91425de21de8e826bd01cc09be2bc723a45f0 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 7 Jun 2016 22:15:40 +0200 Subject: [PATCH 42/74] 1R1W mode fixed --- tsmpool.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tsmpool.cpp b/tsmpool.cpp index 67507c5..d081eb9 100644 --- a/tsmpool.cpp +++ b/tsmpool.cpp @@ -8,6 +8,7 @@ tsmpool::tsmpool(size_t size, int num) this->ok = 1; this->lowest_read_index = -1; this->write_index = 0; + this->my_read_index = 0; if (pthread_mutex_init(&this->mutex, NULL) != 0) this->ok=0; } @@ -51,7 +52,3 @@ void* tsmpool::get_read_buffer(tsmthread_t* thread) void* to_return = buffers[*actual_read_index]; *actual_read_index=index_next(*actual_read_index); } - -void* tsmpool::set_read_index_distance(tsmthread_t* thread, int distance) -{ -} From c4b3527490425c5043e836eca7f78addd32ccdd1 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 21 Jun 2016 00:14:03 +0200 Subject: [PATCH 43/74] Added convert_s24_f and convert_f_s24 --- csdr.c | 46 +++++++++++++++++++++++++++++++++++++--------- libcsdr.c | 37 +++++++++++++++++++++++++++++++++++++ libcsdr.h | 3 +++ 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/csdr.c b/csdr.c index 52de1ab..16f488e 100644 --- a/csdr.c +++ b/csdr.c @@ -61,6 +61,8 @@ char usage[]= " convert_f_s8\n" " convert_f_s16\n" " convert_s16_f\n" +" convert_f_s24\n" +" convert_s24_f\n" " realpart_cf\n" " clipdetect_ff\n" " limit_ff [max_amplitude]\n" @@ -352,7 +354,7 @@ int main(int argc, char *argv[]) char** fifo_buffers = (char**)malloc(sizeof(char*)*fifo_num_buffers); for(int i=0;i2 && !strcmp(argv[2],"--bigendian"); + if(!sendbufsize(initialize_buffers())) return -2; + for(;;) + { + FEOF_CHECK; + FREAD_R; + convert_f_s24(input_buffer, (unsigned char*)output_buffer, the_bufsize, bigendian); + fwrite(output_buffer, sizeof(unsigned char)*3, the_bufsize, stdout); + TRY_YIELD; + } + } + if(!strcmp(argv[1],"convert_s24_f")) + { + int bigendian = argc>2 && !strcmp(argv[2],"--bigendian"); + if(!sendbufsize(initialize_buffers())) return -2; + for(;;) + { + FEOF_CHECK; + fread(input_buffer, sizeof(unsigned char)*3, the_bufsize, stdin); + convert_s24_f((unsigned char*)input_buffer, output_buffer, the_bufsize, bigendian); + FWRITE_R; + TRY_YIELD; + } + } if(!strcmp(argv[1],"realpart_cf")) { if(!sendbufsize(initialize_buffers())) return -2; diff --git a/libcsdr.c b/libcsdr.c index e3b01bf..896fd89 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -999,6 +999,43 @@ void convert_f_s16(float* input, short* output, int input_size) void convert_i16_f(short* input, float* output, int input_size) { convert_s16_f(input, output, input_size); } void convert_f_i16(float* input, short* output, int input_size) { convert_f_s16(input, output, input_size); } +void convert_f_s24(float* input, unsigned char* output, int input_size, int bigendian) +{ + int k=0; + if(bigendian) for(int i=0;i Date: Tue, 21 Jun 2016 00:14:40 +0200 Subject: [PATCH 44/74] Fixed help message --- csdr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csdr.c b/csdr.c index 16f488e..930cd8f 100644 --- a/csdr.c +++ b/csdr.c @@ -61,8 +61,8 @@ char usage[]= " convert_f_s8\n" " convert_f_s16\n" " convert_s16_f\n" -" convert_f_s24\n" -" convert_s24_f\n" +" convert_f_s24 [--bigendian]\n" +" convert_s24_f [--bigendian]\n" " realpart_cf\n" " clipdetect_ff\n" " limit_ff [max_amplitude]\n" From de110ff7197f89e5c48b0f5bfd18798e8f5113a6 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 21 Jun 2016 00:23:43 +0200 Subject: [PATCH 45/74] Fixed 24 bit conversions --- libcsdr.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libcsdr.c b/libcsdr.c index 896fd89..1c77b6c 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -1004,19 +1004,19 @@ void convert_f_s24(float* input, unsigned char* output, int input_size, int bige int k=0; if(bigendian) for(int i=0;i>8); unsigned char* ptemp=(unsigned char*)&temp; - output[k++]=*ptemp; - output[k++]=*(ptemp+1); output[k++]=*(ptemp+2); + output[k++]=*(ptemp+1); + output[k++]=*ptemp; } else for(int i=0;i>8); unsigned char* ptemp=(unsigned char*)&temp; - output[k++]=*(ptemp+2); - output[k++]=*(ptemp+1); output[k++]=*ptemp; + output[k++]=*(ptemp+1); + output[k++]=*(ptemp+2); } } From ef39d8dc276b1fb378c4a6c96be4a7658ddbaf78 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 23 Jun 2016 10:50:08 +0200 Subject: [PATCH 46/74] Fixed 24 bit conversions --- csdr.c | 14 ++++++++------ libcsdr.c | 16 ++++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/csdr.c b/csdr.c index 930cd8f..ec08e27 100644 --- a/csdr.c +++ b/csdr.c @@ -502,26 +502,28 @@ int main(int argc, char *argv[]) } if(!strcmp(argv[1],"convert_f_s24")) { - int bigendian = argc>2 && !strcmp(argv[2],"--bigendian"); + int bigendian = (argc>2) && (!strcmp(argv[2],"--bigendian")); + unsigned char* s24buffer = (unsigned char*)malloc(sizeof(unsigned char)*the_bufsize*3); if(!sendbufsize(initialize_buffers())) return -2; for(;;) { FEOF_CHECK; FREAD_R; - convert_f_s24(input_buffer, (unsigned char*)output_buffer, the_bufsize, bigendian); - fwrite(output_buffer, sizeof(unsigned char)*3, the_bufsize, stdout); + convert_f_s24(input_buffer, s24buffer, the_bufsize, bigendian); + fwrite(s24buffer, sizeof(unsigned char)*3, the_bufsize, stdout); TRY_YIELD; } } if(!strcmp(argv[1],"convert_s24_f")) { - int bigendian = argc>2 && !strcmp(argv[2],"--bigendian"); + int bigendian = (argc>2) && (!strcmp(argv[2],"--bigendian")); + unsigned char* s24buffer = (unsigned char*)malloc(sizeof(unsigned char)*the_bufsize*3); if(!sendbufsize(initialize_buffers())) return -2; for(;;) { FEOF_CHECK; - fread(input_buffer, sizeof(unsigned char)*3, the_bufsize, stdin); - convert_s24_f((unsigned char*)input_buffer, output_buffer, the_bufsize, bigendian); + fread(s24buffer, sizeof(unsigned char)*3, the_bufsize, stdin); + convert_s24_f(s24buffer, output_buffer, the_bufsize, bigendian); FWRITE_R; TRY_YIELD; } diff --git a/libcsdr.c b/libcsdr.c index 1c77b6c..737a786 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -1006,17 +1006,17 @@ void convert_f_s24(float* input, unsigned char* output, int input_size, int bige { int temp=input[i]*(INT_MAX>>8); unsigned char* ptemp=(unsigned char*)&temp; - output[k++]=*(ptemp+2); - output[k++]=*(ptemp+1); output[k++]=*ptemp; + output[k++]=*(ptemp+1); + output[k++]=*(ptemp+2); } else for(int i=0;i>8); unsigned char* ptemp=(unsigned char*)&temp; - output[k++]=*ptemp; - output[k++]=*(ptemp+1); output[k++]=*(ptemp+2); + output[k++]=*(ptemp+1); + output[k++]=*ptemp; } } @@ -1025,13 +1025,13 @@ void convert_s24_f(unsigned char* input, float* output, int input_size, int bige int k=0; if(bigendian) for(int i=0;i Date: Sun, 24 Jul 2016 12:15:11 +0200 Subject: [PATCH 47/74] Fixed TOTAL_MEMORY for emcc --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4bcd3f8..ee16308 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ emcc-get-deps: emmake make; \ emmake make install emcc: - emcc -O3 -Isdr.js/$(FFTW_PACKAGE)/api -Lsdr.js/$(FFTW_PACKAGE)/emscripten-lib -o sdr.js/sdrjs-compiled.js fft_fftw.c libcsdr_wrapper.c -DLIBCSDR_GPL -DUSE_IMA_ADPCM -DUSE_FFTW -lfftw3f -s EXPORTED_FUNCTIONS="`python sdr.js/exported_functions.py`" + emcc -O3 -Isdr.js/$(FFTW_PACKAGE)/api -Lsdr.js/$(FFTW_PACKAGE)/emscripten-lib -o sdr.js/sdrjs-compiled.js fft_fftw.c libcsdr_wrapper.c -s TOTAL_MEMORY=67108864 -DLIBCSDR_GPL -DUSE_IMA_ADPCM -DUSE_FFTW -lfftw3f -s EXPORTED_FUNCTIONS="`python sdr.js/exported_functions.py`" cat sdr.js/sdrjs-header.js sdr.js/sdrjs-compiled.js sdr.js/sdrjs-footer.js > sdr.js/sdr.js emcc-beautify: bash -c 'type js-beautify >/dev/null 2>&1; if [ $$? -eq 0 ]; then js-beautify sdr.js/sdr.js >sdr.js/sdr.js.beautiful; mv sdr.js/sdr.js.beautiful sdr.js/sdr.js; fi' From 8952449b0907d4b23c35619102c176759cb0fe4f Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Mon, 19 Sep 2016 13:15:07 +0200 Subject: [PATCH 48/74] Fixing bug: was using -msse4 for sse4a instead of -msse4a --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ee16308..73f21d8 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ LIBSOURCES = fft_fftw.c libcsdr_wrapper.c #SOURCES = csdr.c $(LIBSOURCES) cpufeature = $(if $(findstring $(1),$(shell cat /proc/cpuinfo)),$(2)) -PARAMS_SSE = $(call cpufeature,sse,-msse) $(call cpufeature,sse2,-msse2) $(call cpufeature,sse3,-msse3) $(call cpufeature,sse4,-msse4) $(call cpufeature,sse4_1,-msse4.1) $(call cpufeature,sse4_2,-msse4.2) -mfpmath=sse +PARAMS_SSE = $(call cpufeature,sse,-msse) $(call cpufeature,sse2,-msse2) $(call cpufeature,sse3,-msse3) $(call cpufeature,sse4a,-msse4a) $(call cpufeature,sse4_1,-msse4.1) $(call cpufeature,sse4_2,-msse4.2 -msse4) -mfpmath=sse PARAMS_NEON = -mfloat-abi=hard -march=armv7-a -mtune=cortex-a8 -mfpu=neon -mvectorize-with-neon-quad -funsafe-math-optimizations -Wformat=0 -DNEON_OPTS #tnx Jan Szumiec for the Raspberry Pi support PARAMS_RASPI = -mfloat-abi=hard -mcpu=arm1176jzf-s -mfpu=vfp -funsafe-math-optimizations -Wformat=0 From f33a05250c79fc0d3da1fd9976cc9e96c136fa90 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 20 Sep 2016 22:37:06 +0200 Subject: [PATCH 49/74] Before starting to work on nmux --- ddcd.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ddcd.cpp b/ddcd.cpp index 9a76d56..5c32522 100644 --- a/ddcd.cpp +++ b/ddcd.cpp @@ -205,7 +205,7 @@ int main(int argc, char* argv[]) if(index_in_current_write_buffer >= bufsize) { current_write_buffer = pool->get_write_buffer(); - index_in_current_write_buffer = 0;error_exiterror_exit + index_in_current_write_buffer = 0; } int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); if(retval>0) @@ -285,6 +285,14 @@ void* client_thread (void* param) //!TODO me_the_client->status = CS_THREAD_RUNNING; char ctl_data_buffer; int retval; + tsmpool* p1_temp; + tsmpool* p2_temp; + const int num_client_buffers = 20; + if(ddc_method == M_TD) + { + p1_temp = new tsmpool(bufsize, ) + } + for(;;) { do @@ -312,7 +320,7 @@ void* client_thread (void* param) //!TODO void error_exit(const char* why) { perror(why); //do we need a \n at the end of (why)? - exit(1); + exit(1); } void print_exit(const char* why) From 11d639b7a346f74142fc1aebf842c76827cc2919 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 20 Sep 2016 23:52:58 +0200 Subject: [PATCH 50/74] Added nmux --- Makefile | 2 + nmux.cpp | 308 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ nmux.h | 27 +++++ tsmpool.h | 5 +- 4 files changed, 340 insertions(+), 2 deletions(-) create mode 100644 nmux.cpp create mode 100644 nmux.h diff --git a/Makefile b/Makefile index 8294e8f..668d318 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,8 @@ csdr: csdr.c libcsdr.so gcc -std=gnu99 $(PARAMS_LOOPVECT) $(PARAMS_SIMD) csdr.c $(PARAMS_LIBS) -L. -lcsdr $(PARAMS_MISC) -o csdr ddcd: ddcd.cpp libcsdr.so ddcd.h g++ $(PARAMS_LOOPVECT) $(PARAMS_SIMD) ddcd.cpp $(PARAMS_LIBS) -L. -lcsdr -lpthread $(PARAMS_MISC) -o ddcd +nmux: nmux.cpp libcsdr.so nmux.h + g++ $(PARAMS_LOOPVECT) $(PARAMS_SIMD) nmux.cpp $(PARAMS_LIBS) -L. -lcsdr -lpthread $(PARAMS_MISC) -o nmux arm-cross: clean-vect #note: this doesn't work since having added FFTW arm-linux-gnueabihf-gcc -std=gnu99 -O3 -fshort-double -ffast-math -dumpbase dumpvect-arm -fdump-tree-vect-details -mfloat-abi=softfp -march=armv7-a -mtune=cortex-a9 -mfpu=neon -mvectorize-with-neon-quad -Wno-unused-result -Wformat=0 $(SOURCES) -lm -o ./csdr diff --git a/nmux.cpp b/nmux.cpp new file mode 100644 index 0000000..2f86278 --- /dev/null +++ b/nmux.cpp @@ -0,0 +1,308 @@ +/* +This software is part of libcsdr, a set of simple DSP routines for +Software Defined Radio. + +Copyright (c) 2014, Andras Retzler +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL ANDRAS RETZLER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "tsmpool.h" + +int host_port = 0; +char host_address[100] = "127.0.0.1"; +int thread_cntr = 0; + +//CLI parameters +int bufsize = 1024; //! currently unused +int bufcnt = 1024; + +char* global_argv; +int global_argc; +tsmpool* pool; + +void sig_handler(int signo) +{ + fprintf(stderr, MSG_START "signal %d caught, exiting...\n", signo); + fflush(stderr); + exit(0); +} + +int main(int argc, char* argv[]) +{ + global_argv = argv; + global_argc = argc; + int c; + for(;;) + { + int option_index = 0; + static struct option long_options[] = { + {"port", required_argument, 0, 'p' }, + {"address", required_argument, 0, 'a' }, + {"bufsize", required_argument, 0, 'b' }, + {"bufcnt", required_argument, 0, 'n' } + }; + c = getopt_long(argc, argv, "p:a:b:n:", long_options, &option_index); + if(c==-1) break; + switch (c) + { + case 'a': + host_address[100-1]=0; + strncpy(host_address,optarg,100-1); + break; + case 'p': + host_port=atoi(optarg); + break; + case 'b': + bufsize=atoi(optarg); + break; + case 'n': + bufcnt=atoi(optarg); + break; + case 0: + case '?': + case ':': + default:; + print_exit(MSG_START "error in getopt_long()\n"); + } + } + + if(!host_port) print_exit(MSG_START "missing required command line argument, --port.\n"); + if(bufsize<0) print_exit(MSG_START "invalid value for --bufsize (should be >0)\n"); + if(bufcnt<0) print_exit(MSG_START "invalid value for --bufcnt (should be >0)\n"); + + //set signals + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sig_handler; + sigaction(SIGTERM, &sa, NULL); + sigaction(SIGKILL, &sa, NULL); + sigaction(SIGQUIT, &sa, NULL); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGHUP, &sa, NULL); + + struct sockaddr_in addr_host; + int listen_socket; + std::vector clients; + clients.reserve(100); + listen_socket=socket(AF_INET,SOCK_STREAM,0); + + int sockopt = 1; + if( setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&sockopt, sizeof(sockopt)) == -1 ) + error_exit(MSG_START "cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 + + memset(&addr_host,'0',sizeof(addr_host)); + addr_host.sin_family = AF_INET; + addr_host.sin_port = htons(host_port); + addr_host.sin_addr.s_addr = INADDR_ANY; + + if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) + error_exit(MSG_START "invalid host address"); + + if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) + error_exit(MSG_START "cannot bind() address to the socket"); + + if( listen(listen_socket, 10) == -1 ) + error_exit(MSG_START "cannot listen() on socket"); + + fprintf(stderr,MSG_START "listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); + + struct sockaddr_in addr_cli; + socklen_t addr_cli_len = sizeof(addr_cli); + int new_socket; + + int highfd = 0; + FD_ZERO(&select_fds); + FD_SET(listen_socket, &select_fds); + maxfd(&highfd, listen_socket); + FD_SET(input_fd, &select_fds); + maxfd(&highfd, input_fd); + + //Set stdin and listen_socket to non-blocking + if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) + error_exit(MSG_START "cannot set_nonblocking()"); + + //Create tsmpool + pool = new tsmpool(bufsize, bufcnt); + if(!pool->ok) print_exit(MSG_START "tsmpool failed to initialize\n"); + + unsigned char* current_write_buffer = pool->get_write_buffer(); + int index_in_current_write_buffer = 0; + + pthread_cond_t* wait_condition = new wait_condition; + pthread_cond_init(wait_condition, NULL); + + for(;;) + { + //Let's wait until there is any new data to read, or any new connection! + select(highfd, &select_fds, NULL, NULL, NULL); + + //Is there a new client connection? + if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) + { + clients_close_all_finished(); + + //We're the parent, let's create a new client and initialize it + client_t* new_client = new client_t; + new_client->error = 0; + memcpy(&new_client->addr, &addr_cli, sizeof(new_client->addr)); + new_client->socket = new_socket; + new_client->status = CS_CREATED; + new_client->tsmthread = pool->register_thread(); + new_client->lpool = pool; + pthread_mutex_init(&new_client->wait_mutex, NULL); + new_client->wait_condition = wait_condition; + new_client->sleeping = 0; + if(pthread_create(&new_client->thread, NULL, client_thread , (void*)&new_client)<0) + { + clients.push_back(new_client); + fprintf(stderr, MSG_START "pthread_create() done, clients now: %d\n", clients.size()); + } + else + { + + fprintf(stderr, MSG_START "pthread_create() failed.\n"); + } + } + + if(index_in_current_write_buffer >= bufsize) + { + current_write_buffer = pool->get_write_buffer(); + pthread_cond_broadcast(wait_condition); + index_in_current_write_buffer = 0; + } + int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); + if(retval>0) + { + index_in_current_write_buffer += retval; + } + else if(retval==0) + { + //!end of input stream, close clients and exit + print_exit(MSG_START "end of input, exiting.\n") + } + } +} + +#if 0 +for (int i=0; ipipefd[1], buf, retval)==-1) + { + + if(!clients[i]->error) + { + print_client(clients[i], "lost buffer, failed to write pipe."); + clients[i]->error=1; + } + //fprintf(stderr, MSG_START "errno is %d\n", errno); //usually 11 + //int wpstatus; + //int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); + //fprintf(stderr, MSG_START "pid is %d\n",clients[i]->pid); + //perror("somethings wrong"); + //if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); + //else if(wpresult == 0) + waitpid(clients[i]->pid, NULL, WNOHANG); + if(!proc_exists(clients[i]->pid)) + { + //Client exited! + print_client(clients[i], "closing client from main process."); + close(clients[i]->pipefd[1]); + close(clients[i]->socket); + delete clients[i]; + clients.erase(clients.begin()+i); + fprintf(stderr, MSG_START "done closing client from main process.\n"); + } + } + else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } +} +} +//TODO: at the end, server closes pipefd[1] for client +#endif + +void client_erase(client_t* client) +{ + pthread_mutex_destroy(client->wait_mutex); + pthread_cond_destroy(client->wait_condition); + pool-> +} + +void clients_close_all_finished() +{ + for(int i=0;istatus == CS_THREAD_FINISHED) + { + client_erase(clients[i]); + clients.erase(i); + } + } +} + +void* client_thread (void* param) +{ + client_t* this_client = (client_t*)param; + this_client->status = CS_THREAD_RUNNING; + int retval; + tsmpool* lpool = this_client->lpool; + for(;;) + { + //wait until there is any data in the tsmpool for me (wait for the server process to wake me up) + for(;;) + { + char* pool_read_buffer = (char*)lpool->get_read_buffer(); + if(pool_read_buffer) break; + pthread_mutex_lock(&this_client->wait_mutex); + this_client->sleeping = 1; + pthread_cond_wait(this_client->wait_condition, &this_client->wait_mutex); + } + + //wait for the socket to be available for write + //read data from server global tsmpool + //write data to client socket + //have an exit condition + + + } + this_client->status = CS_THREAD_FINISHED; + pthread_exit(NULL); + return NULL; +} + +void error_exit(const char* why) +{ + perror(why); //do we need a \n at the end of (why)? + exit(1); +} + +void print_exit(const char* why) +{ + fprintf(stderr, "%s", why); + exit(1); +} + +void maxfd(int* maxfd, int fd) +{ + if(fd>=*maxfd) *maxfd=fd+1; +} diff --git a/nmux.h b/nmux.h new file mode 100644 index 0000000..886baf2 --- /dev/null +++ b/nmux.h @@ -0,0 +1,27 @@ +#include +#include + +#define MSG_START "nmux: " + +typedef enum client_status_e +{ + CS_CREATED, + CS_THREAD_RUNNING, + CS_THREAD_FINISHED +} client_status_t; + + +typedef struct client_s +{ + struct sockaddr_in addr; + int socket; + int error; //set to non-zero on error (data transfer failed) + pthread_t thread; + tsmthread_t tsmthread; + client_status_t status; + //the following members are there to give access to some global variables inside the thread: + tsmpool* lpool; //local pool + int sleeping; + pthread_cond_t* wait_condition; + pthread_mutex_t wait_mutex; +} client_t; diff --git a/tsmpool.h b/tsmpool.h index f4901cc..e89d965 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -26,8 +26,9 @@ public: size_t get_size(); tsmpool(size_t size, int num); void* get_write_buffer(); - int register_thread(); - void* get_read_buffer(int thread_id); + tsmthread_t* register_thread(); + int remove_thread(tsmthread_t* thread); + void* get_read_buffer(tsmthread_t* thread); int index_next(int index) { return (index+1==size)?0:index; } int index_before(int index) { return (index-1<0)?size-1:index; } } From 38d567d96ef41b57c05cea453c07eb46d89c6a7d Mon Sep 17 00:00:00 2001 From: Tatu Peltola Date: Sat, 22 Oct 2016 22:16:47 +0300 Subject: [PATCH 51/74] Fix logaveragepower_cf for FFT sizes below 16384 --- csdr.c | 22 +++++++++------------- libcsdr.c | 2 +- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/csdr.c b/csdr.c index 1598735..18160d6 100644 --- a/csdr.c +++ b/csdr.c @@ -1344,27 +1344,23 @@ int main(int argc, char *argv[]) sscanf(argv[3],"%d",&fft_size); sscanf(argv[4],"%d",&avgnumber); - if(!getbufsize()) return -2; //dummy - if(!sendbufsize(initialize_buffers())) return -2; + float *input = malloc(sizeof(float)*2 * fft_size); + float *output = malloc(sizeof(float) * fft_size); - if(fft_size != the_bufsize) return -2; - - //fprintf(stderr, "logaveragepower_cf %f %d=%d %d\n", add_db, fft_size, the_bufsize, avgnumber); - add_db -= 10*log10(avgnumber); + add_db -= 10.0*log10(avgnumber); for(;;) { int i,n; - for(i = 0; i < the_bufsize; i++) { - output_buffer[i] = 0; + for(i = 0; i < fft_size; i++) { + output[i] = 0; } FEOF_CHECK; for(n = 0; n < avgnumber; n++) { - FREAD_C; - //fprintf(stderr, "averaged %d\n", n); - accumulate_power_cf((complexf*)input_buffer, output_buffer, the_bufsize); + fread (input, sizeof(float)*2, fft_size, stdin); + accumulate_power_cf((complexf*)input, output, fft_size); } - log_ff(NULL, output_buffer, the_bufsize, add_db); - FWRITE_R; + log_ff(output, output, fft_size, add_db); + fwrite (output, sizeof(float), fft_size, stdout); TRY_YIELD; } return 0; diff --git a/libcsdr.c b/libcsdr.c index f76bc8e..e38e15e 100644 --- a/libcsdr.c +++ b/libcsdr.c @@ -979,7 +979,7 @@ void accumulate_power_cf(complexf* input, float* output, int size) } void log_ff(float* input, float* output, int size, float add_db) { - for(int i=0;i Date: Tue, 10 Jan 2017 10:34:42 +0100 Subject: [PATCH 52/74] Started to work on nmux today --- Makefile | 2 +- nmux.cpp | 10 +++++----- nmux.h | 6 +++++- tsmpool.h | 6 +++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 668d318..c9bafbe 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ PARAMS_MISC = -Wno-unused-result FFTW_PACKAGE = fftw-3.3.3 .PHONY: clean-vect clean -all: csdr ddcd +all: csdr nmux libcsdr.so: fft_fftw.c fft_rpi.c libcsdr_wrapper.c libcsdr.c libcsdr_gpl.c fastddc.c fastddc.h fft_fftw.h fft_rpi.h ima_adpcm.h libcsdr_gpl.h libcsdr.h predefined.h @echo NOTE: you may have to manually edit Makefile to optimize for your CPU \(especially if you compile on ARM, please edit PARAMS_NEON\). @echo Auto-detected optimization parameters: $(PARAMS_SIMD) diff --git a/nmux.cpp b/nmux.cpp index 2f86278..0b093b4 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -28,7 +28,7 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "tsmpool.h" +#include "nmux.h" int host_port = 0; char host_address[100] = "127.0.0.1"; @@ -38,7 +38,7 @@ int thread_cntr = 0; int bufsize = 1024; //! currently unused int bufcnt = 1024; -char* global_argv; +char** global_argv; int global_argc; tsmpool* pool; @@ -150,8 +150,9 @@ int main(int argc, char* argv[]) unsigned char* current_write_buffer = pool->get_write_buffer(); int index_in_current_write_buffer = 0; - pthread_cond_t* wait_condition = new wait_condition; - pthread_cond_init(wait_condition, NULL); + pthread_cond_t* wait_condition = new pthread_cond_t; + if(!pthread_cond_init(wait_condition, NULL)) + print_exit(MSG_START "pthread_cond_init failed"); //cond_attrs is ignored by Linux for(;;) { @@ -181,7 +182,6 @@ int main(int argc, char* argv[]) } else { - fprintf(stderr, MSG_START "pthread_create() failed.\n"); } } diff --git a/nmux.h b/nmux.h index 886baf2..cb8df09 100644 --- a/nmux.h +++ b/nmux.h @@ -1,5 +1,9 @@ #include -#include +#include +#include +#include +#include +#include "tsmpool.h" #define MSG_START "nmux: " diff --git a/tsmpool.h b/tsmpool.h index e89d965..f207689 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -3,6 +3,10 @@ //It implements a big circular buffer that one thread writes into, and multiple threads read from. //The reader threads have lower priority than the writer thread (they can be left behind if the don't read fast enough). +#include + +using namespace std; + typedef struct tsmthread_s { int read_index; //it always points to the next buffer to be read @@ -31,4 +35,4 @@ public: void* get_read_buffer(tsmthread_t* thread); int index_next(int index) { return (index+1==size)?0:index; } int index_before(int index) { return (index-1<0)?size-1:index; } -} +}; From 3ad4d159455d6d9f24134acc5ea8b680d25cfc9d Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 10 Jan 2017 20:21:30 +0100 Subject: [PATCH 53/74] Finished nmux, now working on making it compile --- ddcd.h | 1 + nmux.cpp | 109 ++++++++++++++++++++++++++++------------------------ nmux.h | 13 +++++++ tsmpool.cpp | 14 +++++-- tsmpool.h | 9 +++-- 5 files changed, 88 insertions(+), 58 deletions(-) diff --git a/ddcd.h b/ddcd.h index 766be47..f1cf384 100644 --- a/ddcd.h +++ b/ddcd.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/nmux.cpp b/nmux.cpp index 0b093b4..4225678 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -35,7 +35,7 @@ char host_address[100] = "127.0.0.1"; int thread_cntr = 0; //CLI parameters -int bufsize = 1024; //! currently unused +int bufsize = 1024; int bufcnt = 1024; char** global_argv; @@ -133,6 +133,8 @@ int main(int argc, char* argv[]) int new_socket; int highfd = 0; + int input_fd = STDIN_FILENO; + fd_set select_fds; FD_ZERO(&select_fds); FD_SET(listen_socket, &select_fds); maxfd(&highfd, listen_socket); @@ -150,7 +152,10 @@ int main(int argc, char* argv[]) unsigned char* current_write_buffer = pool->get_write_buffer(); int index_in_current_write_buffer = 0; - pthread_cond_t* wait_condition = new pthread_cond_t; + //Create wait condition: client threads waiting for input data from the main thread will be + // waiting on this condition. They will be woken up with pthread_cond_broadcast() if new + // data arrives. + pthread_cond_t* wait_condition = new pthread_cond_t; if(!pthread_cond_init(wait_condition, NULL)) print_exit(MSG_START "pthread_cond_init failed"); //cond_attrs is ignored by Linux @@ -189,7 +194,11 @@ int main(int argc, char* argv[]) if(index_in_current_write_buffer >= bufsize) { current_write_buffer = pool->get_write_buffer(); - pthread_cond_broadcast(wait_condition); + pthread_cond_broadcast(wait_condition); + //Shouldn't we do it after we put data in? + // No, on get_write_buffer() actually the previous buffer is getting available + // for read for threads that wait for new data (wait on pthead mutex + // client->wait_condition). index_in_current_write_buffer = 0; } int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); @@ -199,53 +208,17 @@ int main(int argc, char* argv[]) } else if(retval==0) { - //!end of input stream, close clients and exit - print_exit(MSG_START "end of input, exiting.\n") + //End of input stream, close clients and exit + print_exit(MSG_START "end of input stream, exiting.\n") } } } -#if 0 -for (int i=0; ipipefd[1], buf, retval)==-1) - { - - if(!clients[i]->error) - { - print_client(clients[i], "lost buffer, failed to write pipe."); - clients[i]->error=1; - } - //fprintf(stderr, MSG_START "errno is %d\n", errno); //usually 11 - //int wpstatus; - //int wpresult = waitpid(clients[i]->pid, &wpstatus, WNOHANG); - //fprintf(stderr, MSG_START "pid is %d\n",clients[i]->pid); - //perror("somethings wrong"); - //if(wpresult == -1) print_client(clients[i], "error while waitpid()!"); - //else if(wpresult == 0) - waitpid(clients[i]->pid, NULL, WNOHANG); - if(!proc_exists(clients[i]->pid)) - { - //Client exited! - print_client(clients[i], "closing client from main process."); - close(clients[i]->pipefd[1]); - close(clients[i]->socket); - delete clients[i]; - clients.erase(clients.begin()+i); - fprintf(stderr, MSG_START "done closing client from main process.\n"); - } - } - else { if(clients[i]->error) print_client(clients[i], "pipe okay again."); clients[i]->error=0; } -} -} -//TODO: at the end, server closes pipefd[1] for client -#endif - void client_erase(client_t* client) { pthread_mutex_destroy(client->wait_mutex); pthread_cond_destroy(client->wait_condition); - pool-> + pool->remove_thread(client->tsmthread); } void clients_close_all_finished() @@ -266,30 +239,66 @@ void* client_thread (void* param) this_client->status = CS_THREAD_RUNNING; int retval; tsmpool* lpool = this_client->lpool; + + fd_set client_select_fds; + int client_highfd = 0; + FD_ZERO(&client_select_fds); + FD_SET(client->socket, &client_select_fds); + maxfd(&client_highfd, client->socket); + + //Set client->socket to non-blocking + if(set_nonblocking(client->socket)) + error_exit(MSG_START "cannot set_nonblocking() on client->socket"); + + int client_buffer_index = 0; + char* pool_read_buffer = NULL; + for(;;) { - //wait until there is any data in the tsmpool for me (wait for the server process to wake me up) - for(;;) + //Wait until there is any data to send. + // If I haven't sent all the data from my last buffer, don't wait. + // (Wait for the server process to wake me up.) + while(!pool_read_buffer || client_buffer_index == lpool->size) { char* pool_read_buffer = (char*)lpool->get_read_buffer(); - if(pool_read_buffer) break; pthread_mutex_lock(&this_client->wait_mutex); this_client->sleeping = 1; pthread_cond_wait(this_client->wait_condition, &this_client->wait_mutex); } - //wait for the socket to be available for write - //read data from server global tsmpool - //write data to client socket - //have an exit condition - + //Wait for the socket to be available for write. + select(highfd, NULL, &client_select_fds, NULL, NULL); + //Read data from global tsmpool and write it to client socket + int ret = send(client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, 0); + if(ret == -1) + { + switch(errno) + { + case EAGAIN: break; + default: goto client_thread_exit; + } + } + else client_buffer_index += ret; } + +client_thread_exit: this_client->status = CS_THREAD_FINISHED; + fprintf(stderr, "CS_THREAD_FINISHED"); //Debug pthread_exit(NULL); return NULL; } + +int set_nonblocking(int fd) +{ + int flagtmp; + if((flagtmp = fcntl(fd, F_GETFL))!=-1) + if((flagtmp = fcntl(fd, F_SETFL, flagtmp|O_NONBLOCK))!=-1) + return 0; + return 1; +} + void error_exit(const char* why) { perror(why); //do we need a \n at the end of (why)? diff --git a/nmux.h b/nmux.h index cb8df09..40003ef 100644 --- a/nmux.h +++ b/nmux.h @@ -1,5 +1,9 @@ #include #include +#include +#include +#include +#include #include #include #include @@ -29,3 +33,12 @@ typedef struct client_s pthread_cond_t* wait_condition; pthread_mutex_t wait_mutex; } client_t; + +void print_exit(const char* why); +void sig_handler(int signo); +void client_erase(client_t* client); +void clients_close_all_finished(); +void* client_thread (void* param); +void error_exit(const char* why); +void maxfd(int* maxfd, int fd); +int set_nonblocking(int fd); diff --git a/tsmpool.cpp b/tsmpool.cpp index d081eb9..ff3b773 100644 --- a/tsmpool.cpp +++ b/tsmpool.cpp @@ -1,18 +1,24 @@ #include "tsmpool.h" tsmpool::tsmpool(size_t size, int num) + size(size), + num(num) //number of buffers of (size) to alloc { this->threads_cntr = 0; - this->size = size; - this->num = num; //number of buffers of (size) to alloc this->ok = 1; this->lowest_read_index = -1; this->write_index = 0; this->my_read_index = 0; - if (pthread_mutex_init(&this->mutex, NULL) != 0) this->ok=0; + if (pthread_mutex_init(&this->mutex, NULL) != 0) { this->ok = 0; return; } + for(int i=0; iok = 0; return; } + buffers.push_back(newptr); + } } -size_t tsmpool::get_size() { return this->size; } +int tsmpool::is_ok() { return this->ok; } void* tsmpool::get_write_buffer() { diff --git a/tsmpool.h b/tsmpool.h index f207689..c1ff956 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -15,19 +15,20 @@ typedef struct tsmthread_s class tsmpool { private: - size_t size; - int num; vector threads; vector buffers; int threads_cntr; pthread_mutex_t mutex; - int ok; + int ok; //tsmpool is expected to be included in C-style programs. + // If something fails in the constructor, it will be seen here instead of a try{}catch{} int write_index; //it always points to the next buffer to be written int lowest_read_index; //unused int my_read_index; //it is used when tsmpool is used as a single writer - single reader circular buffer public: - size_t get_size(); + const size_t size; + const int num; + int is_ok(); tsmpool(size_t size, int num); void* get_write_buffer(); tsmthread_t* register_thread(); From 3941ce49e7f0fc12f2a350528733b9e67a1cbb3c Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Wed, 11 Jan 2017 10:48:25 +0100 Subject: [PATCH 54/74] nmux compiles, also fixed a segfault related to getopt --- Makefile | 13 ++++---- nmux.cpp | 87 ++++++++++++++++++++++++++++++----------------------- nmux.h | 6 ++-- tsmpool.cpp | 4 +-- tsmpool.h | 1 + 5 files changed, 64 insertions(+), 47 deletions(-) diff --git a/Makefile b/Makefile index c9bafbe..06f0709 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - LIBSOURCES = fft_fftw.c libcsdr_wrapper.c #SOURCES = csdr.c $(LIBSOURCES) cpufeature = $(if $(findstring $(1),$(shell cat /proc/cpuinfo)),$(2)) @@ -41,6 +39,8 @@ PARAMS_LOOPVECT = -O3 -ffast-math -fdump-tree-vect-details -dumpbase dumpvect PARAMS_LIBS = -g -lm -lrt -lfftw3f -DUSE_FFTW -DLIBCSDR_GPL -DUSE_IMA_ADPCM PARAMS_SO = -fpic PARAMS_MISC = -Wno-unused-result +#DEBUG_ON = 0 #debug is always on by now (anyway it could be compiled with `make DEBUG_ON=1`) +#PARAMS_DEBUG = $(if $(DEBUG_ON),-g,) FFTW_PACKAGE = fftw-3.3.3 .PHONY: clean-vect clean @@ -56,20 +56,21 @@ csdr: csdr.c libcsdr.so gcc -std=gnu99 $(PARAMS_LOOPVECT) $(PARAMS_SIMD) csdr.c $(PARAMS_LIBS) -L. -lcsdr $(PARAMS_MISC) -o csdr ddcd: ddcd.cpp libcsdr.so ddcd.h g++ $(PARAMS_LOOPVECT) $(PARAMS_SIMD) ddcd.cpp $(PARAMS_LIBS) -L. -lcsdr -lpthread $(PARAMS_MISC) -o ddcd -nmux: nmux.cpp libcsdr.so nmux.h - g++ $(PARAMS_LOOPVECT) $(PARAMS_SIMD) nmux.cpp $(PARAMS_LIBS) -L. -lcsdr -lpthread $(PARAMS_MISC) -o nmux +nmux: nmux.cpp libcsdr.so nmux.h tsmpool.cpp tsmpool.h + g++ $(PARAMS_LOOPVECT) $(PARAMS_SIMD) nmux.cpp tsmpool.cpp $(PARAMS_LIBS) -L. -lcsdr -lpthread $(PARAMS_MISC) -o nmux arm-cross: clean-vect #note: this doesn't work since having added FFTW arm-linux-gnueabihf-gcc -std=gnu99 -O3 -fshort-double -ffast-math -dumpbase dumpvect-arm -fdump-tree-vect-details -mfloat-abi=softfp -march=armv7-a -mtune=cortex-a9 -mfpu=neon -mvectorize-with-neon-quad -Wno-unused-result -Wformat=0 $(SOURCES) -lm -o ./csdr clean-vect: rm -f dumpvect*.vect clean: clean-vect - rm -f libcsdr.so csdr ddcd + rm -f libcsdr.so csdr ddcd nmux install: all install -m 0755 libcsdr.so /usr/lib install -m 0755 csdr /usr/bin install -m 0755 csdr-fm /usr/bin - install -m 0755 ddcd /usr/bin + install -m 0755 nmux /usr/bin + -install -m 0755 ddcd /usr/bin ldconfig uninstall: rm /usr/lib/libcsdr.so /usr/bin/csdr /usr/bin/csdr-fm diff --git a/nmux.cpp b/nmux.cpp index 4225678..7a3ee3e 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -30,6 +30,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "nmux.h" +char help_text[]="nmux is a TCP stream multiplexer. It reads data from the standard input, and sends it to each client connected through TCP sockets. Available command line options are:\n" +"\t--port (-p), --address (-a): TCP port and address to listen.\n" +"\t--bufsize (-b), --bufcnt (-n): Internal buffer size and count.\n" +"\t--help (-h): Show this message.\n"; + int host_port = 0; char host_address[100] = "127.0.0.1"; int thread_cntr = 0; @@ -54,6 +59,7 @@ int main(int argc, char* argv[]) global_argv = argv; global_argc = argc; int c; + int no_options = 1; for(;;) { int option_index = 0; @@ -61,10 +67,13 @@ int main(int argc, char* argv[]) {"port", required_argument, 0, 'p' }, {"address", required_argument, 0, 'a' }, {"bufsize", required_argument, 0, 'b' }, - {"bufcnt", required_argument, 0, 'n' } + {"bufcnt", required_argument, 0, 'n' }, + {"help", no_argument, 0, 'h' }, + {0, 0, 0, 0 } }; - c = getopt_long(argc, argv, "p:a:b:n:", long_options, &option_index); + c = getopt_long(argc, argv, "p:a:b:n:h", long_options, &option_index); if(c==-1) break; + no_options = 0; switch (c) { case 'a': @@ -80,14 +89,18 @@ int main(int argc, char* argv[]) case 'n': bufcnt=atoi(optarg); break; + case 'h': + print_exit(help_text); + break; case 0: case '?': case ':': - default:; - print_exit(MSG_START "error in getopt_long()\n"); + default: + print_exit(MSG_START "(main thread) error in getopt_long()\n"); } } + if(no_options) print_exit(help_text); if(!host_port) print_exit(MSG_START "missing required command line argument, --port.\n"); if(bufsize<0) print_exit(MSG_START "invalid value for --bufsize (should be >0)\n"); if(bufcnt<0) print_exit(MSG_START "invalid value for --bufcnt (should be >0)\n"); @@ -110,7 +123,7 @@ int main(int argc, char* argv[]) int sockopt = 1; if( setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&sockopt, sizeof(sockopt)) == -1 ) - error_exit(MSG_START "cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 + error_exit(MSG_START "(main thread) cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 memset(&addr_host,'0',sizeof(addr_host)); addr_host.sin_family = AF_INET; @@ -118,15 +131,15 @@ int main(int argc, char* argv[]) addr_host.sin_addr.s_addr = INADDR_ANY; if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) - error_exit(MSG_START "invalid host address"); + error_exit(MSG_START "(main thread) invalid host address"); if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) - error_exit(MSG_START "cannot bind() address to the socket"); + error_exit(MSG_START "(main thread) cannot bind() address to the socket"); if( listen(listen_socket, 10) == -1 ) - error_exit(MSG_START "cannot listen() on socket"); + error_exit(MSG_START "(main thread) cannot listen() on socket"); - fprintf(stderr,MSG_START "listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); + fprintf(stderr,MSG_START "(main thread) listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); struct sockaddr_in addr_cli; socklen_t addr_cli_len = sizeof(addr_cli); @@ -143,13 +156,13 @@ int main(int argc, char* argv[]) //Set stdin and listen_socket to non-blocking if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) - error_exit(MSG_START "cannot set_nonblocking()"); + error_exit(MSG_START "(main thread) cannot set_nonblocking()"); //Create tsmpool pool = new tsmpool(bufsize, bufcnt); - if(!pool->ok) print_exit(MSG_START "tsmpool failed to initialize\n"); + if(!pool->is_ok()) print_exit(MSG_START "(main thread) tsmpool failed to initialize\n"); - unsigned char* current_write_buffer = pool->get_write_buffer(); + unsigned char* current_write_buffer = (unsigned char*)pool->get_write_buffer(); int index_in_current_write_buffer = 0; //Create wait condition: client threads waiting for input data from the main thread will be @@ -157,7 +170,7 @@ int main(int argc, char* argv[]) // data arrives. pthread_cond_t* wait_condition = new pthread_cond_t; if(!pthread_cond_init(wait_condition, NULL)) - print_exit(MSG_START "pthread_cond_init failed"); //cond_attrs is ignored by Linux + print_exit(MSG_START "(main thread) pthread_cond_init failed"); //cond_attrs is ignored by Linux for(;;) { @@ -167,7 +180,15 @@ int main(int argc, char* argv[]) //Is there a new client connection? if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) { - clients_close_all_finished(); + //Close all finished clients + for(int i=0;istatus == CS_THREAD_FINISHED) + { + client_erase(clients[i]); + clients.erase(clients.begin()+i); + } + } //We're the parent, let's create a new client and initialize it client_t* new_client = new client_t; @@ -183,17 +204,17 @@ int main(int argc, char* argv[]) if(pthread_create(&new_client->thread, NULL, client_thread , (void*)&new_client)<0) { clients.push_back(new_client); - fprintf(stderr, MSG_START "pthread_create() done, clients now: %d\n", clients.size()); + fprintf(stderr, MSG_START "(main thread) pthread_create() done, clients now: %d\n", clients.size()); } else { - fprintf(stderr, MSG_START "pthread_create() failed.\n"); + fprintf(stderr, MSG_START "(main thread) pthread_create() failed.\n"); } } if(index_in_current_write_buffer >= bufsize) { - current_write_buffer = pool->get_write_buffer(); + current_write_buffer = (unsigned char*)pool->get_write_buffer(); pthread_cond_broadcast(wait_condition); //Shouldn't we do it after we put data in? // No, on get_write_buffer() actually the previous buffer is getting available @@ -209,30 +230,22 @@ int main(int argc, char* argv[]) else if(retval==0) { //End of input stream, close clients and exit - print_exit(MSG_START "end of input stream, exiting.\n") + print_exit(MSG_START "(main thread/for) end input stream, exiting.\n"); + } + else if(retval==-1) + { + error_exit(MSG_START "(main thread/for) error in read(), exiting.\n"); } } } void client_erase(client_t* client) { - pthread_mutex_destroy(client->wait_mutex); + pthread_mutex_destroy(&client->wait_mutex); pthread_cond_destroy(client->wait_condition); pool->remove_thread(client->tsmthread); } -void clients_close_all_finished() -{ - for(int i=0;istatus == CS_THREAD_FINISHED) - { - client_erase(clients[i]); - clients.erase(i); - } - } -} - void* client_thread (void* param) { client_t* this_client = (client_t*)param; @@ -243,11 +256,11 @@ void* client_thread (void* param) fd_set client_select_fds; int client_highfd = 0; FD_ZERO(&client_select_fds); - FD_SET(client->socket, &client_select_fds); - maxfd(&client_highfd, client->socket); + FD_SET(this_client->socket, &client_select_fds); + maxfd(&client_highfd, this_client->socket); //Set client->socket to non-blocking - if(set_nonblocking(client->socket)) + if(set_nonblocking(this_client->socket)) error_exit(MSG_START "cannot set_nonblocking() on client->socket"); int client_buffer_index = 0; @@ -260,17 +273,17 @@ void* client_thread (void* param) // (Wait for the server process to wake me up.) while(!pool_read_buffer || client_buffer_index == lpool->size) { - char* pool_read_buffer = (char*)lpool->get_read_buffer(); + char* pool_read_buffer = (char*)lpool->get_read_buffer(this_client->tsmthread); pthread_mutex_lock(&this_client->wait_mutex); this_client->sleeping = 1; pthread_cond_wait(this_client->wait_condition, &this_client->wait_mutex); } //Wait for the socket to be available for write. - select(highfd, NULL, &client_select_fds, NULL, NULL); + select(client_highfd, NULL, &client_select_fds, NULL, NULL); //Read data from global tsmpool and write it to client socket - int ret = send(client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, 0); + int ret = send(this_client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, 0); if(ret == -1) { switch(errno) diff --git a/nmux.h b/nmux.h index 40003ef..f12db48 100644 --- a/nmux.h +++ b/nmux.h @@ -4,6 +4,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -25,7 +28,7 @@ typedef struct client_s int socket; int error; //set to non-zero on error (data transfer failed) pthread_t thread; - tsmthread_t tsmthread; + tsmthread_t* tsmthread; client_status_t status; //the following members are there to give access to some global variables inside the thread: tsmpool* lpool; //local pool @@ -37,7 +40,6 @@ typedef struct client_s void print_exit(const char* why); void sig_handler(int signo); void client_erase(client_t* client); -void clients_close_all_finished(); void* client_thread (void* param); void error_exit(const char* why); void maxfd(int* maxfd, int fd); diff --git a/tsmpool.cpp b/tsmpool.cpp index ff3b773..a82fd7a 100644 --- a/tsmpool.cpp +++ b/tsmpool.cpp @@ -1,6 +1,6 @@ #include "tsmpool.h" -tsmpool::tsmpool(size_t size, int num) +tsmpool::tsmpool(size_t size, int num) : size(size), num(num) //number of buffers of (size) to alloc { @@ -45,7 +45,7 @@ int tsmpool::remove_thread(tsmthread_t* thread) if(threads[i] == thread) { delete threads[i]; - threads.erase(i); + threads.erase(threads.begin()+i); break; } pthread_mutex_unlock(&this->mutex); diff --git a/tsmpool.h b/tsmpool.h index c1ff956..b2ca44a 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -4,6 +4,7 @@ //The reader threads have lower priority than the writer thread (they can be left behind if the don't read fast enough). #include +#include using namespace std; From 47084804dab618ac3f63b5afa71543df90250984 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 12 Jan 2017 15:27:48 +0100 Subject: [PATCH 55/74] nmux: Fixed some bugs and added a lot of debug info --- nmux.cpp | 27 +++++++++++++++++++++++---- nmux.h | 2 +- tsmpool.cpp | 12 +++++++++--- tsmpool.h | 4 ++-- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index 7a3ee3e..56f650b 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -169,22 +169,26 @@ int main(int argc, char* argv[]) // waiting on this condition. They will be woken up with pthread_cond_broadcast() if new // data arrives. pthread_cond_t* wait_condition = new pthread_cond_t; - if(!pthread_cond_init(wait_condition, NULL)) + if(pthread_cond_init(wait_condition, NULL)) print_exit(MSG_START "(main thread) pthread_cond_init failed"); //cond_attrs is ignored by Linux for(;;) { + fprintf(stderr, "mainfor: selecting..."); //Let's wait until there is any new data to read, or any new connection! select(highfd, &select_fds, NULL, NULL, NULL); + fprintf(stderr, "selected.\n"); //Is there a new client connection? if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) { + fprintf(stderr, "mainfor: accepted.\n"); //Close all finished clients for(int i=0;istatus == CS_THREAD_FINISHED) { + fprintf(stderr, "mainfor: client removed: %d\n", i); client_erase(clients[i]); clients.erase(clients.begin()+i); } @@ -193,7 +197,7 @@ int main(int argc, char* argv[]) //We're the parent, let's create a new client and initialize it client_t* new_client = new client_t; new_client->error = 0; - memcpy(&new_client->addr, &addr_cli, sizeof(new_client->addr)); + memcpy(&new_client->addr, &addr_cli, sizeof(struct sockaddr_in)); new_client->socket = new_socket; new_client->status = CS_CREATED; new_client->tsmthread = pool->register_thread(); @@ -201,7 +205,7 @@ int main(int argc, char* argv[]) pthread_mutex_init(&new_client->wait_mutex, NULL); new_client->wait_condition = wait_condition; new_client->sleeping = 0; - if(pthread_create(&new_client->thread, NULL, client_thread , (void*)&new_client)<0) + if(pthread_create(&new_client->thread, NULL, client_thread , (void*)&new_client)==0) { clients.push_back(new_client); fprintf(stderr, MSG_START "(main thread) pthread_create() done, clients now: %d\n", clients.size()); @@ -209,6 +213,9 @@ int main(int argc, char* argv[]) else { fprintf(stderr, MSG_START "(main thread) pthread_create() failed.\n"); + pool->remove_thread(new_client->tsmthread); + pthread_mutex_destroy(&new_client->wait_mutex); + delete new_client; } } @@ -223,6 +230,7 @@ int main(int argc, char* argv[]) index_in_current_write_buffer = 0; } int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); + fprintf(stderr, "mainfor: read %d\n", retval); if(retval>0) { index_in_current_write_buffer += retval; @@ -248,16 +256,22 @@ void client_erase(client_t* client) void* client_thread (void* param) { + fprintf(stderr, "client 0x%x: started!\n", (unsigned)param); client_t* this_client = (client_t*)param; this_client->status = CS_THREAD_RUNNING; int retval; tsmpool* lpool = this_client->lpool; + fprintf(stderr, "client 0x%x: make maxfd... ", (unsigned)param); fd_set client_select_fds; + fprintf(stderr, "1"); int client_highfd = 0; FD_ZERO(&client_select_fds); + fprintf(stderr, "2 %d <= %d ", this_client->socket, FD_SETSIZE); FD_SET(this_client->socket, &client_select_fds); + fprintf(stderr, "3"); maxfd(&client_highfd, this_client->socket); + fprintf(stderr, "done.\n"); //Set client->socket to non-blocking if(set_nonblocking(this_client->socket)) @@ -271,8 +285,9 @@ void* client_thread (void* param) //Wait until there is any data to send. // If I haven't sent all the data from my last buffer, don't wait. // (Wait for the server process to wake me up.) - while(!pool_read_buffer || client_buffer_index == lpool->size) + while(!pool_read_buffer || client_buffer_index >= lpool->size) { + fprintf(stderr, "client 0x%x: waiting\n", (unsigned)param); char* pool_read_buffer = (char*)lpool->get_read_buffer(this_client->tsmthread); pthread_mutex_lock(&this_client->wait_mutex); this_client->sleeping = 1; @@ -280,10 +295,14 @@ void* client_thread (void* param) } //Wait for the socket to be available for write. + fprintf(stderr, "client 0x%x: selecting...", (unsigned)param); select(client_highfd, NULL, &client_select_fds, NULL, NULL); + fprintf(stderr, "done.\n"); //Read data from global tsmpool and write it to client socket + fprintf(stderr, "client 0x%x: sending...", (unsigned)param); int ret = send(this_client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, 0); + fprintf(stderr, "done.\n"); if(ret == -1) { switch(errno) diff --git a/nmux.h b/nmux.h index f12db48..8efaf5a 100644 --- a/nmux.h +++ b/nmux.h @@ -31,7 +31,7 @@ typedef struct client_s tsmthread_t* tsmthread; client_status_t status; //the following members are there to give access to some global variables inside the thread: - tsmpool* lpool; //local pool + tsmpool* lpool; int sleeping; pthread_cond_t* wait_condition; pthread_mutex_t wait_mutex; diff --git a/tsmpool.cpp b/tsmpool.cpp index a82fd7a..92d94cb 100644 --- a/tsmpool.cpp +++ b/tsmpool.cpp @@ -8,7 +8,7 @@ tsmpool::tsmpool(size_t size, int num) : this->ok = 1; this->lowest_read_index = -1; this->write_index = 0; - this->my_read_index = 0; + this->my_read_index = index_before(0); if (pthread_mutex_init(&this->mutex, NULL) != 0) { this->ok = 0; return; } for(int i=0; iok; } void* tsmpool::get_write_buffer() { //if(write_index==index_before(lowest_read_index)) return NULL; + pthread_mutex_lock(&this->mutex); void* to_return = buffers[write_index]; - write_index=index_next(write_index); + write_index = index_next(write_index); + pthread_mutex_unlock(&this->mutex); + return to_return; } tsmthread_t* tsmpool::register_thread() { if(!ok) return NULL; pthread_mutex_lock(&this->mutex); - tsmthread_t* thread = new tsmthread_t; + tsmthread_t* thread = new tsmthread_t(); thread->read_index = index_before(write_index); threads.push_back(thread); pthread_mutex_unlock(&this->mutex); @@ -53,8 +56,11 @@ int tsmpool::remove_thread(tsmthread_t* thread) void* tsmpool::get_read_buffer(tsmthread_t* thread) { + pthread_mutex_lock(&this->mutex); int* actual_read_index = (thread==NULL) ? &my_read_index : &thread->read_index; if(*actual_read_index==index_before(write_index)) return NULL; void* to_return = buffers[*actual_read_index]; *actual_read_index=index_next(*actual_read_index); + pthread_mutex_unlock(&this->mutex); + return to_return; } diff --git a/tsmpool.h b/tsmpool.h index b2ca44a..26f3197 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -35,6 +35,6 @@ public: tsmthread_t* register_thread(); int remove_thread(tsmthread_t* thread); void* get_read_buffer(tsmthread_t* thread); - int index_next(int index) { return (index+1==size)?0:index; } - int index_before(int index) { return (index-1<0)?size-1:index; } + int index_next(int index) { return (index+1==num)?0:index+1; } + int index_before(int index) { return (index-1<0)?num-1:index-1; } }; From c2058aa34e7b78e47aa4c5d049b1a45554c696f2 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 12 Jan 2017 16:24:59 +0100 Subject: [PATCH 56/74] nmux: client now uses poll; more messages are printed if NMUX_DEBUG is on --- nmux.cpp | 76 ++++++++++++++++++++++++++++---------------------------- nmux.h | 2 ++ 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index 56f650b..e2e200e 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -96,7 +96,7 @@ int main(int argc, char* argv[]) case '?': case ':': default: - print_exit(MSG_START "(main thread) error in getopt_long()\n"); + print_exit(MSG_START "error in getopt_long()\n"); } } @@ -123,7 +123,7 @@ int main(int argc, char* argv[]) int sockopt = 1; if( setsockopt(listen_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&sockopt, sizeof(sockopt)) == -1 ) - error_exit(MSG_START "(main thread) cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 + error_exit(MSG_START "cannot set SO_REUSEADDR"); //the best description on SO_REUSEADDR ever: http://stackoverflow.com/a/14388707/3182453 memset(&addr_host,'0',sizeof(addr_host)); addr_host.sin_family = AF_INET; @@ -131,15 +131,15 @@ int main(int argc, char* argv[]) addr_host.sin_addr.s_addr = INADDR_ANY; if( (addr_host.sin_addr.s_addr=inet_addr(host_address)) == INADDR_NONE ) - error_exit(MSG_START "(main thread) invalid host address"); + error_exit(MSG_START "invalid host address"); if( bind(listen_socket, (struct sockaddr*) &addr_host, sizeof(addr_host)) < 0 ) - error_exit(MSG_START "(main thread) cannot bind() address to the socket"); + error_exit(MSG_START "cannot bind() address to the socket"); if( listen(listen_socket, 10) == -1 ) - error_exit(MSG_START "(main thread) cannot listen() on socket"); + error_exit(MSG_START "cannot listen() on socket"); - fprintf(stderr,MSG_START "(main thread) listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); + fprintf(stderr, MSG_START "listening on %s:%d\n", inet_ntoa(addr_host.sin_addr), host_port); struct sockaddr_in addr_cli; socklen_t addr_cli_len = sizeof(addr_cli); @@ -156,11 +156,11 @@ int main(int argc, char* argv[]) //Set stdin and listen_socket to non-blocking if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) - error_exit(MSG_START "(main thread) cannot set_nonblocking()"); + error_exit(MSG_START "cannot set_nonblocking()"); //Create tsmpool pool = new tsmpool(bufsize, bufcnt); - if(!pool->is_ok()) print_exit(MSG_START "(main thread) tsmpool failed to initialize\n"); + if(!pool->is_ok()) print_exit(MSG_START "tsmpool failed to initialize\n"); unsigned char* current_write_buffer = (unsigned char*)pool->get_write_buffer(); int index_in_current_write_buffer = 0; @@ -170,25 +170,25 @@ int main(int argc, char* argv[]) // data arrives. pthread_cond_t* wait_condition = new pthread_cond_t; if(pthread_cond_init(wait_condition, NULL)) - print_exit(MSG_START "(main thread) pthread_cond_init failed"); //cond_attrs is ignored by Linux + print_exit(MSG_START "pthread_cond_init failed"); //cond_attrs is ignored by Linux for(;;) { - fprintf(stderr, "mainfor: selecting..."); + if(NMUX_DEBUG) fprintf(stderr, "mainfor: selecting..."); //Let's wait until there is any new data to read, or any new connection! select(highfd, &select_fds, NULL, NULL, NULL); - fprintf(stderr, "selected.\n"); + if(NMUX_DEBUG) fprintf(stderr, "selected.\n"); //Is there a new client connection? if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) { - fprintf(stderr, "mainfor: accepted.\n"); + if(NMUX_DEBUG) fprintf(stderr, "mainfor: accepted (%d).\n", new_socket); //Close all finished clients for(int i=0;istatus == CS_THREAD_FINISHED) { - fprintf(stderr, "mainfor: client removed: %d\n", i); + if(NMUX_DEBUG) fprintf(stderr, "mainfor: client removed: %d\n", i); client_erase(clients[i]); clients.erase(clients.begin()+i); } @@ -205,14 +205,14 @@ int main(int argc, char* argv[]) pthread_mutex_init(&new_client->wait_mutex, NULL); new_client->wait_condition = wait_condition; new_client->sleeping = 0; - if(pthread_create(&new_client->thread, NULL, client_thread , (void*)&new_client)==0) + if(pthread_create(&new_client->thread, NULL, client_thread, (void*)&new_client)==0) { clients.push_back(new_client); - fprintf(stderr, MSG_START "(main thread) pthread_create() done, clients now: %d\n", clients.size()); + fprintf(stderr, MSG_START "pthread_create() done, clients now: %d\n", clients.size()); } else { - fprintf(stderr, MSG_START "(main thread) pthread_create() failed.\n"); + fprintf(stderr, MSG_START "pthread_create() failed.\n"); pool->remove_thread(new_client->tsmthread); pthread_mutex_destroy(&new_client->wait_mutex); delete new_client; @@ -230,7 +230,7 @@ int main(int argc, char* argv[]) index_in_current_write_buffer = 0; } int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); - fprintf(stderr, "mainfor: read %d\n", retval); + if(NMUX_DEBUG) fprintf(stderr, "mainfor: read %d\n", retval); if(retval>0) { index_in_current_write_buffer += retval; @@ -261,23 +261,21 @@ void* client_thread (void* param) this_client->status = CS_THREAD_RUNNING; int retval; tsmpool* lpool = this_client->lpool; + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: socket = %d!\n", (unsigned)param, this_client->socket); - fprintf(stderr, "client 0x%x: make maxfd... ", (unsigned)param); - fd_set client_select_fds; - fprintf(stderr, "1"); - int client_highfd = 0; - FD_ZERO(&client_select_fds); - fprintf(stderr, "2 %d <= %d ", this_client->socket, FD_SETSIZE); - FD_SET(this_client->socket, &client_select_fds); - fprintf(stderr, "3"); - maxfd(&client_highfd, this_client->socket); - fprintf(stderr, "done.\n"); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: poll init...", (unsigned)param); + struct pollfd pollfds[1]; + pollfds[0].fd = this_client->socket; + pollfds[0].events = POLLOUT; + pollfds[0].revents = 0; + if(NMUX_DEBUG) fprintf(stderr, "done.\n"); - //Set client->socket to non-blocking + //Set this_client->socket to non-blocking if(set_nonblocking(this_client->socket)) - error_exit(MSG_START "cannot set_nonblocking() on client->socket"); + error_exit(MSG_START "cannot set_nonblocking() on this_client->socket"); int client_buffer_index = 0; + int client_goto_source = 0; char* pool_read_buffer = NULL; for(;;) @@ -287,7 +285,7 @@ void* client_thread (void* param) // (Wait for the server process to wake me up.) while(!pool_read_buffer || client_buffer_index >= lpool->size) { - fprintf(stderr, "client 0x%x: waiting\n", (unsigned)param); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: cond_waiting for more data\n", (unsigned)param); char* pool_read_buffer = (char*)lpool->get_read_buffer(this_client->tsmthread); pthread_mutex_lock(&this_client->wait_mutex); this_client->sleeping = 1; @@ -295,20 +293,22 @@ void* client_thread (void* param) } //Wait for the socket to be available for write. - fprintf(stderr, "client 0x%x: selecting...", (unsigned)param); - select(client_highfd, NULL, &client_select_fds, NULL, NULL); - fprintf(stderr, "done.\n"); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: polling for socket write...", (unsigned)param); + int ret = poll(pollfds, 1, -1); + if(NMUX_DEBUG) fprintf(stderr, "done.\n"); + if(ret == 0) continue; + else if (ret == -1) { client_goto_source = 1; goto client_thread_exit; } //Read data from global tsmpool and write it to client socket - fprintf(stderr, "client 0x%x: sending...", (unsigned)param); - int ret = send(this_client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, 0); - fprintf(stderr, "done.\n"); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: sending...", (unsigned)param); + ret = send(this_client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, 0); + if(NMUX_DEBUG) fprintf(stderr, "done.\n"); if(ret == -1) { switch(errno) { case EAGAIN: break; - default: goto client_thread_exit; + default: client_goto_source = 2; goto client_thread_exit; } } else client_buffer_index += ret; @@ -316,7 +316,7 @@ void* client_thread (void* param) client_thread_exit: this_client->status = CS_THREAD_FINISHED; - fprintf(stderr, "CS_THREAD_FINISHED"); //Debug + fprintf(stderr, "client 0x%x: CS_THREAD_FINISHED, client_goto_source = %d, errno = %d", (unsigned)param, client_goto_source, errno); pthread_exit(NULL); return NULL; } diff --git a/nmux.h b/nmux.h index 8efaf5a..c8d875c 100644 --- a/nmux.h +++ b/nmux.h @@ -7,12 +7,14 @@ #include #include #include +#include #include #include #include #include "tsmpool.h" #define MSG_START "nmux: " +#define NMUX_DEBUG 1 typedef enum client_status_e { From 1177e036608e44191abcb9d9182beced85acf915 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 12 Jan 2017 16:56:54 +0100 Subject: [PATCH 57/74] nmux: things start to work --- nmux.cpp | 16 +++++++++------- tsmpool.cpp | 9 ++++++++- tsmpool.h | 1 + 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index e2e200e..77fd52d 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -174,15 +174,15 @@ int main(int argc, char* argv[]) for(;;) { - if(NMUX_DEBUG) fprintf(stderr, "mainfor: selecting..."); + // if(NMUX_DEBUG) fprintf(stderr, "mainfor: selecting..."); //Let's wait until there is any new data to read, or any new connection! select(highfd, &select_fds, NULL, NULL, NULL); - if(NMUX_DEBUG) fprintf(stderr, "selected.\n"); + // if(NMUX_DEBUG) fprintf(stderr, "selected.\n"); //Is there a new client connection? if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) { - if(NMUX_DEBUG) fprintf(stderr, "mainfor: accepted (%d).\n", new_socket); + if(NMUX_DEBUG) fprintf(stderr, "mainfor: accepted (socket = %d).\n", new_socket); //Close all finished clients for(int i=0;iwait_mutex, NULL); new_client->wait_condition = wait_condition; new_client->sleeping = 0; - if(pthread_create(&new_client->thread, NULL, client_thread, (void*)&new_client)==0) + if(pthread_create(&new_client->thread, NULL, client_thread, (void*)new_client)==0) { clients.push_back(new_client); fprintf(stderr, MSG_START "pthread_create() done, clients now: %d\n", clients.size()); @@ -230,7 +230,7 @@ int main(int argc, char* argv[]) index_in_current_write_buffer = 0; } int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); - if(NMUX_DEBUG) fprintf(stderr, "mainfor: read %d\n", retval); + // if(NMUX_DEBUG) fprintf(stderr, "mainfor: read %d\n", retval); if(retval>0) { index_in_current_write_buffer += retval; @@ -242,7 +242,8 @@ int main(int argc, char* argv[]) } else if(retval==-1) { - error_exit(MSG_START "(main thread/for) error in read(), exiting.\n"); + if(errno == EAGAIN) { if(NMUX_DEBUG) fprintf(stderr, "mainfor: read %d\n", retval); } + else error_exit(MSG_START "(main thread/for) error in read(), exiting.\n"); } } } @@ -286,7 +287,8 @@ void* client_thread (void* param) while(!pool_read_buffer || client_buffer_index >= lpool->size) { if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: cond_waiting for more data\n", (unsigned)param); - char* pool_read_buffer = (char*)lpool->get_read_buffer(this_client->tsmthread); + pool_read_buffer = (char*)lpool->get_read_buffer(this_client->tsmthread); + if(pool_read_buffer) { client_buffer_index = 0; break; } pthread_mutex_lock(&this_client->wait_mutex); this_client->sleeping = 1; pthread_cond_wait(this_client->wait_condition, &this_client->wait_mutex); diff --git a/tsmpool.cpp b/tsmpool.cpp index 92d94cb..7334865 100644 --- a/tsmpool.cpp +++ b/tsmpool.cpp @@ -27,6 +27,7 @@ void* tsmpool::get_write_buffer() void* to_return = buffers[write_index]; write_index = index_next(write_index); pthread_mutex_unlock(&this->mutex); + fprintf(stderr, "gwb: write_index = %d\n", write_index); return to_return; } @@ -58,9 +59,15 @@ void* tsmpool::get_read_buffer(tsmthread_t* thread) { pthread_mutex_lock(&this->mutex); int* actual_read_index = (thread==NULL) ? &my_read_index : &thread->read_index; - if(*actual_read_index==index_before(write_index)) return NULL; + if(*actual_read_index==index_before(write_index)) + { + fprintf(stderr, "grb: fail," + "read_index %d is just before write_index\n", *actual_read_index); + return NULL; + } void* to_return = buffers[*actual_read_index]; *actual_read_index=index_next(*actual_read_index); pthread_mutex_unlock(&this->mutex); + fprintf(stderr, "grb: read_index = %d\n", *actual_read_index); return to_return; } diff --git a/tsmpool.h b/tsmpool.h index 26f3197..b825d8a 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -3,6 +3,7 @@ //It implements a big circular buffer that one thread writes into, and multiple threads read from. //The reader threads have lower priority than the writer thread (they can be left behind if the don't read fast enough). +#include #include #include From 20f107b5785c5187ae136e1545d1ac91d2b94aae Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 12 Jan 2017 17:45:59 +0100 Subject: [PATCH 58/74] nmux: fixed unlocked mutex in tsmpool --- nmux.cpp | 23 ++++++++++++++--------- tsmpool.cpp | 7 ++++--- tsmpool.h | 6 +++++- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index 77fd52d..b5dab59 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -102,8 +102,8 @@ int main(int argc, char* argv[]) if(no_options) print_exit(help_text); if(!host_port) print_exit(MSG_START "missing required command line argument, --port.\n"); - if(bufsize<0) print_exit(MSG_START "invalid value for --bufsize (should be >0)\n"); - if(bufcnt<0) print_exit(MSG_START "invalid value for --bufcnt (should be >0)\n"); + if(bufsize=<0) print_exit(MSG_START "invalid value for --bufsize (should be >0)\n"); + if(bufcnt=<0) print_exit(MSG_START "invalid value for --bufcnt (should be >0)\n"); //set signals struct sigaction sa; @@ -174,10 +174,10 @@ int main(int argc, char* argv[]) for(;;) { - // if(NMUX_DEBUG) fprintf(stderr, "mainfor: selecting..."); + if(NMUX_DEBUG) fprintf(stderr, "mainfor: selecting..."); //Let's wait until there is any new data to read, or any new connection! select(highfd, &select_fds, NULL, NULL, NULL); - // if(NMUX_DEBUG) fprintf(stderr, "selected.\n"); + if(NMUX_DEBUG) fprintf(stderr, "selected.\n"); //Is there a new client connection? if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) @@ -221,16 +221,20 @@ int main(int argc, char* argv[]) if(index_in_current_write_buffer >= bufsize) { + if(NMUX_DEBUG) fprintf(stderr, "mainfor: gwbing..."); current_write_buffer = (unsigned char*)pool->get_write_buffer(); + if(NMUX_DEBUG) fprintf(stderr, "gwbed.\nmainfor: cond broadcasting..."); pthread_cond_broadcast(wait_condition); + if(NMUX_DEBUG) fprintf(stderr, "cond broadcasted.\n"); //Shouldn't we do it after we put data in? // No, on get_write_buffer() actually the previous buffer is getting available // for read for threads that wait for new data (wait on pthead mutex // client->wait_condition). index_in_current_write_buffer = 0; } + if(NMUX_DEBUG) fprintf(stderr, "mainfor: reading...\n"); int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); - // if(NMUX_DEBUG) fprintf(stderr, "mainfor: read %d\n", retval); + if(NMUX_DEBUG) fprintf(stderr, "read %d\n", retval); if(retval>0) { index_in_current_write_buffer += retval; @@ -269,7 +273,7 @@ void* client_thread (void* param) pollfds[0].fd = this_client->socket; pollfds[0].events = POLLOUT; pollfds[0].revents = 0; - if(NMUX_DEBUG) fprintf(stderr, "done.\n"); + if(NMUX_DEBUG) fprintf(stderr, "client poll inited.\n"); //Set this_client->socket to non-blocking if(set_nonblocking(this_client->socket)) @@ -286,9 +290,10 @@ void* client_thread (void* param) // (Wait for the server process to wake me up.) while(!pool_read_buffer || client_buffer_index >= lpool->size) { - if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: cond_waiting for more data\n", (unsigned)param); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: trying to grb\n", (unsigned)param); pool_read_buffer = (char*)lpool->get_read_buffer(this_client->tsmthread); if(pool_read_buffer) { client_buffer_index = 0; break; } + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: cond_waiting for more data\n", (unsigned)param); pthread_mutex_lock(&this_client->wait_mutex); this_client->sleeping = 1; pthread_cond_wait(this_client->wait_condition, &this_client->wait_mutex); @@ -297,14 +302,14 @@ void* client_thread (void* param) //Wait for the socket to be available for write. if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: polling for socket write...", (unsigned)param); int ret = poll(pollfds, 1, -1); - if(NMUX_DEBUG) fprintf(stderr, "done.\n"); + if(NMUX_DEBUG) fprintf(stderr, "client polled for socket write.\n"); if(ret == 0) continue; else if (ret == -1) { client_goto_source = 1; goto client_thread_exit; } //Read data from global tsmpool and write it to client socket if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: sending...", (unsigned)param); ret = send(this_client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, 0); - if(NMUX_DEBUG) fprintf(stderr, "done.\n"); + if(NMUX_DEBUG) fprintf(stderr, "client sent.\n"); if(ret == -1) { switch(errno) diff --git a/tsmpool.cpp b/tsmpool.cpp index 7334865..d70e8ba 100644 --- a/tsmpool.cpp +++ b/tsmpool.cpp @@ -27,7 +27,7 @@ void* tsmpool::get_write_buffer() void* to_return = buffers[write_index]; write_index = index_next(write_index); pthread_mutex_unlock(&this->mutex); - fprintf(stderr, "gwb: write_index = %d\n", write_index); + if(TSM_DEBUG) fprintf(stderr, "gwb: write_index = %d\n", write_index); return to_return; } @@ -61,13 +61,14 @@ void* tsmpool::get_read_buffer(tsmthread_t* thread) int* actual_read_index = (thread==NULL) ? &my_read_index : &thread->read_index; if(*actual_read_index==index_before(write_index)) { - fprintf(stderr, "grb: fail," + if(TSM_DEBUG) fprintf(stderr, "grb: fail," "read_index %d is just before write_index\n", *actual_read_index); + pthread_mutex_unlock(&this->mutex); return NULL; } void* to_return = buffers[*actual_read_index]; *actual_read_index=index_next(*actual_read_index); pthread_mutex_unlock(&this->mutex); - fprintf(stderr, "grb: read_index = %d\n", *actual_read_index); + if(TSM_DEBUG) fprintf(stderr, "grb: read_index = %d\n", *actual_read_index); return to_return; } diff --git a/tsmpool.h b/tsmpool.h index b825d8a..daf5592 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -3,10 +3,14 @@ //It implements a big circular buffer that one thread writes into, and multiple threads read from. //The reader threads have lower priority than the writer thread (they can be left behind if the don't read fast enough). -#include #include #include +#define TSM_DEBUG 1 +#if TSM_DEBUG == 1 + #include +#endif + using namespace std; typedef struct tsmthread_s From 8b4323237fe44826c98a4e481234ea271ae87771 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 12 Jan 2017 17:50:51 +0100 Subject: [PATCH 59/74] nmux: fixed cmp op --- nmux.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index b5dab59..ae53e95 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -102,8 +102,8 @@ int main(int argc, char* argv[]) if(no_options) print_exit(help_text); if(!host_port) print_exit(MSG_START "missing required command line argument, --port.\n"); - if(bufsize=<0) print_exit(MSG_START "invalid value for --bufsize (should be >0)\n"); - if(bufcnt=<0) print_exit(MSG_START "invalid value for --bufcnt (should be >0)\n"); + if(bufsize<=0) print_exit(MSG_START "invalid value for --bufsize (should be >0)\n"); + if(bufcnt<=0) print_exit(MSG_START "invalid value for --bufcnt (should be >0)\n"); //set signals struct sigaction sa; From 795a77e7eef842811d5d39602662ec791638806c Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 12 Jan 2017 18:57:23 +0100 Subject: [PATCH 60/74] nmux: rearranged read() code for fixing EAGAINs --- nmux.cpp | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index ae53e95..cc1cd3e 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -232,22 +232,26 @@ int main(int argc, char* argv[]) // client->wait_condition). index_in_current_write_buffer = 0; } - if(NMUX_DEBUG) fprintf(stderr, "mainfor: reading...\n"); - int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); - if(NMUX_DEBUG) fprintf(stderr, "read %d\n", retval); - if(retval>0) + for(;;) { - index_in_current_write_buffer += retval; - } - else if(retval==0) - { - //End of input stream, close clients and exit - print_exit(MSG_START "(main thread/for) end input stream, exiting.\n"); - } - else if(retval==-1) - { - if(errno == EAGAIN) { if(NMUX_DEBUG) fprintf(stderr, "mainfor: read %d\n", retval); } - else error_exit(MSG_START "(main thread/for) error in read(), exiting.\n"); + if(NMUX_DEBUG) fprintf(stderr, "mainfor: reading...\n"); + int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); + if(NMUX_DEBUG) fprintf(stderr, "read %d\n", retval); + if(retval>0) + { + index_in_current_write_buffer += retval; + break; + } + else if(retval==0) + { + //End of input stream, close clients and exit + print_exit(MSG_START "(main thread/for) end input stream, exiting.\n"); + } + else if(retval==-1) + { + if(errno == EAGAIN) { if(NMUX_DEBUG) fprintf(stderr, "mainfor: read EAGAIN\n"); /* seems like select would block forever, so we just read again */ break; } + else error_exit(MSG_START "(main thread/for) error in read(), exiting.\n"); + } } } } From da132294562ed3656a59722676f374e8d8d2b542 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 12 Jan 2017 19:10:27 +0100 Subject: [PATCH 61/74] nmux: fixed how wait conditions and mutexes work, according to man --- nmux.cpp | 24 ++++++++++++------------ nmux.h | 2 -- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index cc1cd3e..4588cb9 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -47,6 +47,9 @@ char** global_argv; int global_argc; tsmpool* pool; +pthread_cond_t wait_condition; +pthread_mutex_t wait_mutex; + void sig_handler(int signo) { fprintf(stderr, MSG_START "signal %d caught, exiting...\n", signo); @@ -168,9 +171,11 @@ int main(int argc, char* argv[]) //Create wait condition: client threads waiting for input data from the main thread will be // waiting on this condition. They will be woken up with pthread_cond_broadcast() if new // data arrives. - pthread_cond_t* wait_condition = new pthread_cond_t; - if(pthread_cond_init(wait_condition, NULL)) + if(pthread_cond_init(&wait_condition, NULL)) print_exit(MSG_START "pthread_cond_init failed"); //cond_attrs is ignored by Linux + + if(pthread_mutex_init(&wait_mutex, NULL)) + print_exit(MSG_START "pthread_mutex_t failed"); //cond_attrs is ignored by Linux for(;;) { @@ -202,8 +207,6 @@ int main(int argc, char* argv[]) new_client->status = CS_CREATED; new_client->tsmthread = pool->register_thread(); new_client->lpool = pool; - pthread_mutex_init(&new_client->wait_mutex, NULL); - new_client->wait_condition = wait_condition; new_client->sleeping = 0; if(pthread_create(&new_client->thread, NULL, client_thread, (void*)new_client)==0) { @@ -214,7 +217,6 @@ int main(int argc, char* argv[]) { fprintf(stderr, MSG_START "pthread_create() failed.\n"); pool->remove_thread(new_client->tsmthread); - pthread_mutex_destroy(&new_client->wait_mutex); delete new_client; } } @@ -224,12 +226,12 @@ int main(int argc, char* argv[]) if(NMUX_DEBUG) fprintf(stderr, "mainfor: gwbing..."); current_write_buffer = (unsigned char*)pool->get_write_buffer(); if(NMUX_DEBUG) fprintf(stderr, "gwbed.\nmainfor: cond broadcasting..."); - pthread_cond_broadcast(wait_condition); + pthread_cond_broadcast(&wait_condition); if(NMUX_DEBUG) fprintf(stderr, "cond broadcasted.\n"); //Shouldn't we do it after we put data in? // No, on get_write_buffer() actually the previous buffer is getting available - // for read for threads that wait for new data (wait on pthead mutex - // client->wait_condition). + // for read for threads that wait for new data (wait on global pthead mutex + // wait_condition). index_in_current_write_buffer = 0; } for(;;) @@ -258,8 +260,6 @@ int main(int argc, char* argv[]) void client_erase(client_t* client) { - pthread_mutex_destroy(&client->wait_mutex); - pthread_cond_destroy(client->wait_condition); pool->remove_thread(client->tsmthread); } @@ -298,9 +298,9 @@ void* client_thread (void* param) pool_read_buffer = (char*)lpool->get_read_buffer(this_client->tsmthread); if(pool_read_buffer) { client_buffer_index = 0; break; } if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: cond_waiting for more data\n", (unsigned)param); - pthread_mutex_lock(&this_client->wait_mutex); + pthread_mutex_lock(&wait_mutex); this_client->sleeping = 1; - pthread_cond_wait(this_client->wait_condition, &this_client->wait_mutex); + pthread_cond_wait(&wait_condition, &wait_mutex); } //Wait for the socket to be available for write. diff --git a/nmux.h b/nmux.h index c8d875c..f59efe0 100644 --- a/nmux.h +++ b/nmux.h @@ -35,8 +35,6 @@ typedef struct client_s //the following members are there to give access to some global variables inside the thread: tsmpool* lpool; int sleeping; - pthread_cond_t* wait_condition; - pthread_mutex_t wait_mutex; } client_t; void print_exit(const char* why); From 2f633f1f29620fcfa3b4dadc2f391abf4dee619b Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 12 Jan 2017 23:32:45 +0100 Subject: [PATCH 62/74] nmux: hacking around select --- nmux.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nmux.cpp b/nmux.cpp index 4588cb9..b0a369c 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -181,12 +181,15 @@ int main(int argc, char* argv[]) { if(NMUX_DEBUG) fprintf(stderr, "mainfor: selecting..."); //Let's wait until there is any new data to read, or any new connection! - select(highfd, &select_fds, NULL, NULL, NULL); + int select_num = select(highfd, &select_fds, NULL, NULL, NULL); if(NMUX_DEBUG) fprintf(stderr, "selected.\n"); + if(select_num == -1) error_exit("mainfor select() error"); + //Is there a new client connection? if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) { + select_num--; if(NMUX_DEBUG) fprintf(stderr, "mainfor: accepted (socket = %d).\n", new_socket); //Close all finished clients for(int i=0;i= bufsize) { if(NMUX_DEBUG) fprintf(stderr, "mainfor: gwbing..."); From ac392f4eadca1da077652bbbf2657c3042d8a2f0 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 12 Jan 2017 23:52:53 +0100 Subject: [PATCH 63/74] nmux: implemented select properly at the end --- nmux.cpp | 71 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index b0a369c..557fa9a 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -149,16 +149,13 @@ int main(int argc, char* argv[]) int new_socket; int highfd = 0; - int input_fd = STDIN_FILENO; - fd_set select_fds; - FD_ZERO(&select_fds); - FD_SET(listen_socket, &select_fds); maxfd(&highfd, listen_socket); - FD_SET(input_fd, &select_fds); - maxfd(&highfd, input_fd); + maxfd(&highfd, STDIN_FILENO); + + fd_set select_fds; //Set stdin and listen_socket to non-blocking - if(set_nonblocking(input_fd) || set_nonblocking(listen_socket)) + if(set_nonblocking(STDIN_FILENO) || set_nonblocking(listen_socket)) error_exit(MSG_START "cannot set_nonblocking()"); //Create tsmpool @@ -179,17 +176,20 @@ int main(int argc, char* argv[]) for(;;) { + FD_ZERO(&select_fds); + FD_SET(listen_socket, &select_fds); + FD_SET(STDIN_FILENO, &select_fds); + if(NMUX_DEBUG) fprintf(stderr, "mainfor: selecting..."); //Let's wait until there is any new data to read, or any new connection! - int select_num = select(highfd, &select_fds, NULL, NULL, NULL); + int select_ret = select(highfd, &select_fds, NULL, NULL, NULL); if(NMUX_DEBUG) fprintf(stderr, "selected.\n"); - - if(select_num == -1) error_exit("mainfor select() error"); + if(select_ret == -1) error_exit("mainfor select() error"); //Is there a new client connection? - if( (new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) + if( FD_ISSET(listen_socket, &select_fds) && ((new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) ) { - select_num--; + select_ret--; if(NMUX_DEBUG) fprintf(stderr, "mainfor: accepted (socket = %d).\n", new_socket); //Close all finished clients for(int i=0;i= bufsize) + if( FD_ISSET(STDIN_FILENO, &select_fds) ) { - if(NMUX_DEBUG) fprintf(stderr, "mainfor: gwbing..."); - current_write_buffer = (unsigned char*)pool->get_write_buffer(); - if(NMUX_DEBUG) fprintf(stderr, "gwbed.\nmainfor: cond broadcasting..."); - pthread_cond_broadcast(&wait_condition); - if(NMUX_DEBUG) fprintf(stderr, "cond broadcasted.\n"); - //Shouldn't we do it after we put data in? - // No, on get_write_buffer() actually the previous buffer is getting available - // for read for threads that wait for new data (wait on global pthead mutex - // wait_condition). - index_in_current_write_buffer = 0; - } - for(;;) - { - if(NMUX_DEBUG) fprintf(stderr, "mainfor: reading...\n"); - int retval = read(input_fd, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); - if(NMUX_DEBUG) fprintf(stderr, "read %d\n", retval); - if(retval>0) + if(index_in_current_write_buffer >= bufsize) { - index_in_current_write_buffer += retval; - break; + if(NMUX_DEBUG) fprintf(stderr, "mainfor: gwbing..."); + current_write_buffer = (unsigned char*)pool->get_write_buffer(); + if(NMUX_DEBUG) fprintf(stderr, "gwbed.\nmainfor: cond broadcasting..."); + pthread_cond_broadcast(&wait_condition); + if(NMUX_DEBUG) fprintf(stderr, "cond broadcasted.\n"); + //Shouldn't we do it after we put data in? + // No, on get_write_buffer() actually the previous buffer is getting available + // for read for threads that wait for new data (wait on global pthead mutex + // wait_condition). + index_in_current_write_buffer = 0; } - else if(retval==0) + + if(NMUX_DEBUG) fprintf(stderr, "mainfor: reading...\n"); + int read_ret = read(STDIN_FILENO, current_write_buffer + index_in_current_write_buffer, bufsize - index_in_current_write_buffer); + if(NMUX_DEBUG) fprintf(stderr, "read %d\n", read_ret); + if(read_ret>0) + { + index_in_current_write_buffer += read_ret; + } + else if(read_ret==0) { //End of input stream, close clients and exit print_exit(MSG_START "(main thread/for) end input stream, exiting.\n"); } - else if(retval==-1) + else if(read_ret==-1) { - if(errno == EAGAIN) { if(NMUX_DEBUG) fprintf(stderr, "mainfor: read EAGAIN\n"); /* seems like select would block forever, so we just read again */ break; } + if(errno == EAGAIN) { if(NMUX_DEBUG) fprintf(stderr, "mainfor: read EAGAIN\n"); /* seems like select would block forever, so we just read again */ } else error_exit(MSG_START "(main thread/for) error in read(), exiting.\n"); } } From dd51b9cc046ecff8144ebe96ff2c6aa268b52b82 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Fri, 13 Jan 2017 12:00:43 +0100 Subject: [PATCH 64/74] nmux: seems to work now --- nmux.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nmux.cpp b/nmux.cpp index 557fa9a..03f3b2f 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -231,7 +231,9 @@ int main(int argc, char* argv[]) if(NMUX_DEBUG) fprintf(stderr, "mainfor: gwbing..."); current_write_buffer = (unsigned char*)pool->get_write_buffer(); if(NMUX_DEBUG) fprintf(stderr, "gwbed.\nmainfor: cond broadcasting..."); + pthread_mutex_lock(&wait_mutex); pthread_cond_broadcast(&wait_condition); + pthread_mutex_unlock(&wait_mutex); if(NMUX_DEBUG) fprintf(stderr, "cond broadcasted.\n"); //Shouldn't we do it after we put data in? // No, on get_write_buffer() actually the previous buffer is getting available @@ -304,6 +306,7 @@ void* client_thread (void* param) pthread_mutex_lock(&wait_mutex); this_client->sleeping = 1; pthread_cond_wait(&wait_condition, &wait_mutex); + pthread_mutex_unlock(&wait_mutex); } //Wait for the socket to be available for write. From 1712c5af3b131dee6d3987e5cdd50d92c6231b62 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Fri, 13 Jan 2017 13:00:48 +0100 Subject: [PATCH 65/74] nmux: MSG_NOSIGNAL fixed client close --- nmux.cpp | 20 +++++++++++--------- nmux.h | 1 - 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index 03f3b2f..49733b9 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -189,7 +189,6 @@ int main(int argc, char* argv[]) //Is there a new client connection? if( FD_ISSET(listen_socket, &select_fds) && ((new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) ) { - select_ret--; if(NMUX_DEBUG) fprintf(stderr, "mainfor: accepted (socket = %d).\n", new_socket); //Close all finished clients for(int i=0;istatus == CS_THREAD_FINISHED) { if(NMUX_DEBUG) fprintf(stderr, "mainfor: client removed: %d\n", i); - client_erase(clients[i]); + //client destructor + pool->remove_thread(clients[i]->tsmthread); clients.erase(clients.begin()+i); + i--; } } + if(NMUX_DEBUG) + { + fprintf(stderr, "\x1b[33mmainfor: clients now: "); + for(int i=0;iremove_thread(client->tsmthread); -} - void* client_thread (void* param) { fprintf(stderr, "client 0x%x: started!\n", (unsigned)param); @@ -318,7 +320,7 @@ void* client_thread (void* param) //Read data from global tsmpool and write it to client socket if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: sending...", (unsigned)param); - ret = send(this_client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, 0); + ret = send(this_client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, MSG_NOSIGNAL); if(NMUX_DEBUG) fprintf(stderr, "client sent.\n"); if(ret == -1) { @@ -332,8 +334,8 @@ void* client_thread (void* param) } client_thread_exit: - this_client->status = CS_THREAD_FINISHED; fprintf(stderr, "client 0x%x: CS_THREAD_FINISHED, client_goto_source = %d, errno = %d", (unsigned)param, client_goto_source, errno); + this_client->status = CS_THREAD_FINISHED; pthread_exit(NULL); return NULL; } diff --git a/nmux.h b/nmux.h index f59efe0..faf95ef 100644 --- a/nmux.h +++ b/nmux.h @@ -39,7 +39,6 @@ typedef struct client_s void print_exit(const char* why); void sig_handler(int signo); -void client_erase(client_t* client); void* client_thread (void* param); void error_exit(const char* why); void maxfd(int* maxfd, int fd); From e4fc05537b4d8da265d3c1af5dd2aac8acdbfa8f Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sun, 15 Jan 2017 20:13:24 +0100 Subject: [PATCH 66/74] nmux: clients print double check before and after client clean up --- nmux.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nmux.cpp b/nmux.cpp index 49733b9..654d068 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -189,6 +189,12 @@ int main(int argc, char* argv[]) //Is there a new client connection? if( FD_ISSET(listen_socket, &select_fds) && ((new_socket = accept(listen_socket, (struct sockaddr*)&addr_cli, &addr_cli_len)) != -1) ) { + if(NMUX_DEBUG) + { + fprintf(stderr, "\x1b[1m\x1b[33mmainfor: clients before closing: "); + for(int i=0;i Date: Sun, 15 Jan 2017 20:38:55 +0100 Subject: [PATCH 67/74] nmux: added todo --- nmux-todo.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 nmux-todo.md diff --git a/nmux-todo.md b/nmux-todo.md new file mode 100644 index 0000000..750ec79 --- /dev/null +++ b/nmux-todo.md @@ -0,0 +1,8 @@ +Try in OpenWebRX +Add UDP support +Evaluate performance against ncat +Remove debug messages +Document in csdr +Test with a limited number of people +Announce on blog + From ada8a692d4d3df14a9e8a11fa17a2eb552cdbd64 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Sun, 15 Jan 2017 21:09:24 +0100 Subject: [PATCH 68/74] csdr: better error message on missing function, nmux: switching debug messages off --- csdr.c | 5 +++-- nmux-todo.md | 5 ++--- nmux.h | 2 +- tsmpool.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/csdr.c b/csdr.c index c1efeaa..28579fe 100644 --- a/csdr.c +++ b/csdr.c @@ -1917,8 +1917,9 @@ int main(int argc, char *argv[]) { return 0; } - - return badsyntax("function name given in argument 1 does not exist. Possible causes:\n- You mistyped the commandline.\n- You need to update csdr to a newer version (if available)."); + + fprintf(stderr,"csdr: function name given in argument 1 (%s) does not exist. Possible causes:\n- You mistyped the commandline.\n- You need to update csdr to a newer version (if available).", argv[1]); + return -1; } diff --git a/nmux-todo.md b/nmux-todo.md index 750ec79..8e011f7 100644 --- a/nmux-todo.md +++ b/nmux-todo.md @@ -1,8 +1,7 @@ +Remove nmux repo, it will rather be part of csdr Try in OpenWebRX Add UDP support Evaluate performance against ncat Remove debug messages -Document in csdr +Document README.md Test with a limited number of people -Announce on blog - diff --git a/nmux.h b/nmux.h index faf95ef..038bc51 100644 --- a/nmux.h +++ b/nmux.h @@ -14,7 +14,7 @@ #include "tsmpool.h" #define MSG_START "nmux: " -#define NMUX_DEBUG 1 +#define NMUX_DEBUG 0 typedef enum client_status_e { diff --git a/tsmpool.h b/tsmpool.h index daf5592..3db486c 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -6,7 +6,7 @@ #include #include -#define TSM_DEBUG 1 +#define TSM_DEBUG 0 #if TSM_DEBUG == 1 #include #endif From 3f3c3b27c872108d6cd16fe6ea2e029075fffd8c Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 19 Jan 2017 17:08:49 +0100 Subject: [PATCH 69/74] Fixed include on TSM_DEBUG --- tsmpool.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tsmpool.h b/tsmpool.h index 3db486c..d24cc4e 100644 --- a/tsmpool.h +++ b/tsmpool.h @@ -7,9 +7,7 @@ #include #define TSM_DEBUG 0 -#if TSM_DEBUG == 1 - #include -#endif +#include using namespace std; From 52f99f680ab6ec740c963e6a590e06272a934e0a Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 19 Jan 2017 19:40:45 +0100 Subject: [PATCH 70/74] nmux: added docs --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index db2816d..e0f2ddc 100644 --- a/README.md +++ b/README.md @@ -515,6 +515,15 @@ To remove *sdr.js* and the compiled dependencies: make emcc-clean +## [nmux] (#nmux) + +The repo also contains a command line tool called `nmux`, which is a TCP stream multiplexer. It reads data from the standard input, and sends it to each client connected through TCP sockets. Available command line options are: +* `--port (-p), --address (-a):` TCP port and address to listen. +* `--bufsize (-b), --bufcnt (-n)`: Internal buffer size and count. +* `--help (-h)`: Show help message. + +`nmux` was originally written for use in OpenWebRX. + ## [Licensing] (#licensing) Most of the code of `libcsdr` is under BSD license. From 6c947a83f5cd55c82d4aa2e169c94c512c94f223 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 19 Jan 2017 19:42:01 +0100 Subject: [PATCH 71/74] Added gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a20ba03 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +csdr +nmux +ddcd +*.o +tags From 338ff5158fd8cdaa87209aa1f668581e62712c36 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 19 Jan 2017 19:43:20 +0100 Subject: [PATCH 72/74] Fixed .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index a20ba03..1d4be5f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ csdr nmux ddcd *.o +*.so tags +dumpvect.*.vect From ccb2c159a44d7c05fafc24fe6890b1495c0651b1 Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Thu, 19 Jan 2017 19:57:21 +0100 Subject: [PATCH 73/74] Updated Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 920bb46..7db181a 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ install: all install -m 0755 csdr /usr/bin install -m 0755 csdr-fm /usr/bin install -m 0755 nmux /usr/bin - -install -m 0755 ddcd /usr/bin + #-install -m 0755 ddcd /usr/bin ldconfig uninstall: rm /usr/lib/libcsdr.so /usr/bin/csdr /usr/bin/csdr-fm From 1339352da70279203dfa08d0069b4ace34b8dbcc Mon Sep 17 00:00:00 2001 From: ha7ilm Date: Tue, 24 Jan 2017 17:49:04 +0100 Subject: [PATCH 74/74] unsigned to intptr_t for build on x64 --- nmux.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nmux.cpp b/nmux.cpp index 654d068..7d75012 100644 --- a/nmux.cpp +++ b/nmux.cpp @@ -192,7 +192,7 @@ int main(int argc, char* argv[]) if(NMUX_DEBUG) { fprintf(stderr, "\x1b[1m\x1b[33mmainfor: clients before closing: "); - for(int i=0;istatus = CS_THREAD_RUNNING; int retval; tsmpool* lpool = this_client->lpool; - if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: socket = %d!\n", (unsigned)param, this_client->socket); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: socket = %d!\n", (intptr_t)param, this_client->socket); - if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: poll init...", (unsigned)param); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: poll init...", (intptr_t)param); struct pollfd pollfds[1]; pollfds[0].fd = this_client->socket; pollfds[0].events = POLLOUT; @@ -307,10 +307,10 @@ void* client_thread (void* param) // (Wait for the server process to wake me up.) while(!pool_read_buffer || client_buffer_index >= lpool->size) { - if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: trying to grb\n", (unsigned)param); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: trying to grb\n", (intptr_t)param); pool_read_buffer = (char*)lpool->get_read_buffer(this_client->tsmthread); if(pool_read_buffer) { client_buffer_index = 0; break; } - if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: cond_waiting for more data\n", (unsigned)param); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: cond_waiting for more data\n", (intptr_t)param); pthread_mutex_lock(&wait_mutex); this_client->sleeping = 1; pthread_cond_wait(&wait_condition, &wait_mutex); @@ -318,14 +318,14 @@ void* client_thread (void* param) } //Wait for the socket to be available for write. - if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: polling for socket write...", (unsigned)param); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: polling for socket write...", (intptr_t)param); int ret = poll(pollfds, 1, -1); if(NMUX_DEBUG) fprintf(stderr, "client polled for socket write.\n"); if(ret == 0) continue; else if (ret == -1) { client_goto_source = 1; goto client_thread_exit; } //Read data from global tsmpool and write it to client socket - if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: sending...", (unsigned)param); + if(NMUX_DEBUG) fprintf(stderr, "client 0x%x: sending...", (intptr_t)param); ret = send(this_client->socket, pool_read_buffer + client_buffer_index, lpool->size - client_buffer_index, MSG_NOSIGNAL); if(NMUX_DEBUG) fprintf(stderr, "client sent.\n"); if(ret == -1) @@ -340,7 +340,7 @@ void* client_thread (void* param) } client_thread_exit: - fprintf(stderr, "client 0x%x: CS_THREAD_FINISHED, client_goto_source = %d, errno = %d", (unsigned)param, client_goto_source, errno); + fprintf(stderr, "client 0x%x: CS_THREAD_FINISHED, client_goto_source = %d, errno = %d", (intptr_t)param, client_goto_source, errno); this_client->status = CS_THREAD_FINISHED; pthread_exit(NULL); return NULL;