[med-svn] r11483 - in trunk/packages/ffindex/trunk/debian: . patches
Laszlo Kajan
lkajan-guest at alioth.debian.org
Wed Jun 27 09:52:43 UTC 2012
Author: lkajan-guest
Date: 2012-06-27 09:52:43 +0000 (Wed, 27 Jun 2012)
New Revision: 11483
Added:
trunk/packages/ffindex/trunk/debian/patches/40000000_bug
trunk/packages/ffindex/trunk/debian/patches/hardening
trunk/packages/ffindex/trunk/debian/patches/num_max_entries_bug
Modified:
trunk/packages/ffindex/trunk/debian/changelog
trunk/packages/ffindex/trunk/debian/patches/Makefile
trunk/packages/ffindex/trunk/debian/patches/series
trunk/packages/ffindex/trunk/debian/rules
Log:
fixed 40000000 bug; index bug; added hardening flags
Modified: trunk/packages/ffindex/trunk/debian/changelog
===================================================================
--- trunk/packages/ffindex/trunk/debian/changelog 2012-06-27 06:01:53 UTC (rev 11482)
+++ trunk/packages/ffindex/trunk/debian/changelog 2012-06-27 09:52:43 UTC (rev 11483)
@@ -1,6 +1,7 @@
ffindex (0.9.6.1-1) unstable; urgency=low
* New upstream release.
+ * Fixed many bugs.
-- Laszlo Kajan <lkajan at rostlab.org> Tue, 26 Jun 2012 11:43:41 +0200
Added: trunk/packages/ffindex/trunk/debian/patches/40000000_bug
===================================================================
--- trunk/packages/ffindex/trunk/debian/patches/40000000_bug (rev 0)
+++ trunk/packages/ffindex/trunk/debian/patches/40000000_bug 2012-06-27 09:52:43 UTC (rev 11483)
@@ -0,0 +1,104 @@
+From: Laszlo Kajan <lkajan at rostlab.org>
+Subject: 40000000 bug
+ The default value 40000000 for FFINDEX_MAX_INDEX_ENTRIES_DEFAULT is too high for a machine with 2G (or less) RAM.
+ Bug fixed by introducing an environment variable FFINDEX_MAX_INDEX_ENTRIES to control the value.
+Forwarded: yes
+
+--- a/src/ffindex.c
++++ b/src/ffindex.c
+@@ -220,7 +220,7 @@
+ ffindex_index_t* ffindex_index_parse(FILE *index_file, size_t num_max_entries)
+ {
+ if(num_max_entries == 0)
+- num_max_entries = FFINDEX_MAX_INDEX_ENTRIES_DEFAULT;
++ num_max_entries = ffindex_max_index_entries();
+ size_t nbytes = sizeof(ffindex_index_t) + (sizeof(ffindex_entry_t) * num_max_entries);
+ ffindex_index_t *index = (ffindex_index_t *)malloc(nbytes);
+ if(index == NULL)
+--- a/src/ffindex.h
++++ b/src/ffindex.h
+@@ -18,6 +18,8 @@
+ #include <stdio.h>
+
+ #define FFINDEX_VERSION 0.961
++// lkajan: with ~72 bytes per entry, the below makes a default allocation over 2G. malloc in ffindex.c:225 fails on a machine with 'only' 2G of RAM.
++// lkajan: let the default remain 40000000, and let the actual value be controlled by the env var FFINDEX_MAX_INDEX_ENTRIES wherever FFINDEX_MAX_INDEX_ENTRIES_DEFAULT is used.
+ #define FFINDEX_MAX_INDEX_ENTRIES_DEFAULT 40000000
+ #define FFINDEX_MAX_ENTRY_NAME_LENTH 56
+
+@@ -41,6 +43,8 @@
+ ffindex_entry_t entries[]; /* This array is as big as the excess memory allocated for this struct. */
+ } ffindex_index_t;
+
++inline size_t ffindex_max_index_entries(){ char *FMIE = getenv("FFINDEX_MAX_INDEX_ENTRIES"); if( FMIE ) { return atol( FMIE ); } else { return FFINDEX_MAX_INDEX_ENTRIES_DEFAULT; } }
++
+ int ffindex_insert_memory(FILE *data_file, FILE *index_file, size_t *offset, char *from_start, size_t from_length, char *name);
+
+ int ffindex_insert_file(FILE *data_file, FILE *index_file, size_t *offset, const char *path, char *name);
+Binary files a/src/ffindex_apply and b/src/ffindex_apply differ
+--- a/src/ffindex_apply.c
++++ b/src/ffindex_apply.c
+@@ -34,7 +34,9 @@
+ if(argn < 4)
+ {
+ fprintf(stderr, "USAGE: ffindex_apply DATA_FILENAME INDEX_FILENAME PROGRAM [PROGRAM_ARGS]*\n"
+- "\nDesigned and implemented by Andy Hauser <hauser at genzentrum.lmu.de>.\n",
++ "ENVIRONMENT\n"
++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n"
++ "\nDesigned and implemented by Andy Hauser <hauser at genzentrum.lmu.de>.\n"
+ );
+ return -1;
+ }
+--- a/src/ffindex_build.c
++++ b/src/ffindex_build.c
+@@ -39,6 +39,8 @@
+ "\t\t-f can be specified up to %d times\n"
+ "\t-s\tsort index file\n"
+ "\t-v\tprint version and other info then exit\n"
++ "ENVIRONMENT\n"
++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n"
+ "\nDesigned and implemented by Andreas W. Hauser <hauser at genzentrum.lmu.de>.\n", MAX_FILENAME_LIST_FILES);
+ }
+
+--- a/src/ffindex_from_fasta.c
++++ b/src/ffindex_from_fasta.c
+@@ -33,6 +33,8 @@
+ {
+ fprintf(stderr, "USAGE: ffindex_from_fasta -v | [-s] data_filename index_filename fasta_filename\n"
+ "\t-s\tsort index file\n"
++ "ENVIRONMENT\n"
++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n"
+ "\nDesigned and implemented by Andreas W. Hauser <hauser at genzentrum.lmu.de>.\n");
+ }
+
+--- a/src/ffindex_get.c
++++ b/src/ffindex_get.c
+@@ -27,6 +27,8 @@
+ if(argn < 3)
+ {
+ fprintf(stderr, "USAGE: ffindex_get data_filename index_filename filename(s)\n"
++ "ENVIRONMENT\n"
++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n"
+ "\nDesigned and implemented by Andreas W. Hauser <hauser at genzentrum.lmu.de>.\n");
+ return -1;
+ }
+--- a/src/ffindex_modify.c
++++ b/src/ffindex_modify.c
+@@ -37,6 +37,8 @@
+ "\t-t\tuse tree (default, will be removed soon)\n"
+ "\t-u\tunlink entry (remove from index only)\n"
+ "\t-v\tprint version and other info then exit\n"
++ "ENVIRONMENT\n"
++ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n"
+ "\nDesigned and implemented by Andreas W. Hauser <hauser at genzentrum.lmu.de>.\n",
+ MAX_FILENAME_LIST_FILES);
+ }
+@@ -125,7 +127,7 @@
+ }
+ else
+ {
+- char** sorted_names_to_unlink = malloc(FFINDEX_MAX_INDEX_ENTRIES_DEFAULT * sizeof(char *));
++ char** sorted_names_to_unlink = malloc(ffindex_max_index_entries() * sizeof(char *));
+ if(sorted_names_to_unlink == NULL)
+ fferror_print(__FILE__, __LINE__, __func__, "malloc failed");
+ /* For each list_file unlink all entries */
Modified: trunk/packages/ffindex/trunk/debian/patches/Makefile
===================================================================
--- trunk/packages/ffindex/trunk/debian/patches/Makefile 2012-06-27 06:01:53 UTC (rev 11482)
+++ trunk/packages/ffindex/trunk/debian/patches/Makefile 2012-06-27 09:52:43 UTC (rev 11483)
@@ -1,6 +1,12 @@
-* make Makefile DESTDIR-aware
-* deinstall -> uninstall
-* fix issues where the path in help pages is taken from the build-time location of the executables
+From: Laszlo Kajan <lkajan at rostlab.org>
+Subject: Makefile fixes
+ * make Makefile DESTDIR-aware
+ * other changes to allow help2man to work better.
+ * deinstall -> uninstall
+ .
+ Upstream is aware but not interested in DESTDIR.
+Forwarded: yes
+
--- a/Makefile
+++ b/Makefile
@@ -6,15 +6,8 @@
@@ -24,7 +30,7 @@
+ $(MAKE) -C src -f $(MFILE) $@
--- a/src/Makefile
+++ b/src/Makefile
-@@ -1,15 +1,19 @@
+@@ -1,15 +1,16 @@
-INSTALL_DIR?=..
-libdir=`([ -d /usr/lib64 ] && echo lib64) || echo lib`
+PREFIX?=/usr
@@ -36,9 +42,6 @@
+CC=gcc
CFLAGS+=-std=c99 -O2 -g -fPIC -Wall -pedantic
-LDFLAGS=-lffindex -L.
-+CFLAGS+=-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security
-+CPPFLAGS+=-D_FORTIFY_SOURCE=2
-+LDFLAGS+=-Wl,-z,relro
+SOCURRENT=0
+SOVERSION=$(SOCURRENT).1.0
ifdef FFINDEX_STATIC
@@ -50,7 +53,7 @@
endif
#-Wl,--version-script=libffindex.map
-@@ -25,33 +29,34 @@
+@@ -25,33 +26,34 @@
ffindex_apply: $(FFINDEX_OBJECTS) ffindex_apply.o ffindex.h
ffindex_apply_mpi: $(FFINDEX_OBJECTS) ffindex_apply_mpi.o ffindex.h
@@ -95,7 +98,7 @@
rm -f /tmp/test.data /tmp/test.ffindex
LD_LIBRARY_PATH=".:$(LD_LIBRARY_PATH)" ./ffindex_build -s /tmp/test.data /tmp/test.ffindex ../test/data ../test/data2
LD_LIBRARY_PATH=".:$(LD_LIBRARY_PATH)" ./ffindex_get /tmp/test.data /tmp/test.ffindex a b foo | tee /tmp/test.out
-@@ -71,9 +76,11 @@
+@@ -71,9 +73,11 @@
LD_LIBRARY_PATH=".:$(LD_LIBRARY_PATH)" zsh -c "time ./ffindex_get ../test_big/nr20_11Jan10_a3m_db ../test_big/nr20_11Jan10_a3m_db.ffindex BABBAMIBA.a3m KARDUZEBA.a3m HARPAQIBA.a3m WOQQATEBA.a3m BEBRAQIBA.a3m GIRSAKEBA.a3m NAQTIFIBA.a3m BIWCIQABA.a3m > /dev/null"
clean:
@@ -108,7 +111,7 @@
install:
mkdir -p $(INSTALL_DIR)/bin
mkdir -p $(INSTALL_DIR)/$(libdir)
-@@ -83,9 +90,10 @@
+@@ -83,9 +87,10 @@
install ffindex_modify $(INSTALL_DIR)/bin/ffindex_modify
install ffindex_from_fasta $(INSTALL_DIR)/bin/ffindex_from_fasta
install ffindex_apply $(INSTALL_DIR)/bin/ffindex_apply
@@ -191,34 +194,31 @@
return -1;
}
char *data_filename = argv[1];
---- a/src/ffindex.h
-+++ b/src/ffindex.h
-@@ -18,7 +18,8 @@
- #include <stdio.h>
-
- #define FFINDEX_VERSION 0.961
--#define FFINDEX_MAX_INDEX_ENTRIES_DEFAULT 40000000
-+// lkajan: with 72 bytes per entry, the below makes a default allocation over 2G. malloc in ffindex.c:225 fails with hardening flags. Let's give a value for approx. 256MB.
-+#define FFINDEX_MAX_INDEX_ENTRIES_DEFAULT 3728270
- #define FFINDEX_MAX_ENTRY_NAME_LENTH 56
-
- enum ffindex_type { PLAIN_FILE, SORTED_FILE, SORTED_ARRAY, TREE };
---- a/src/ffindex.c
-+++ b/src/ffindex.c
-@@ -223,13 +223,14 @@
- num_max_entries = FFINDEX_MAX_INDEX_ENTRIES_DEFAULT;
- size_t nbytes = sizeof(ffindex_index_t) + (sizeof(ffindex_entry_t) * num_max_entries);
- ffindex_index_t *index = (ffindex_index_t *)malloc(nbytes);
-- index->num_max_entries = num_max_entries;
- if(index == NULL)
+--- a/src/ffindex_apply.c
++++ b/src/ffindex_apply.c
+@@ -33,9 +33,9 @@
+ {
+ if(argn < 4)
{
-- fferror_print(__FILE__, __LINE__, __func__, "malloc failed");
-+ char buf[2048]; snprintf( buf, 2048, "malloc failed to allocate %ld bytes", nbytes );
-+ fferror_print(__FILE__, __LINE__, __func__, buf );
- return NULL;
+- fprintf(stderr, "USAGE: %s DATA_FILENAME INDEX_FILENAME PROGRAM [PROGRAM_ARGS]*\n"
++ fprintf(stderr, "USAGE: ffindex_apply DATA_FILENAME INDEX_FILENAME PROGRAM [PROGRAM_ARGS]*\n"
+ "\nDesigned and implemented by Andy Hauser <hauser at genzentrum.lmu.de>.\n",
+- argv[0]);
++ );
+ return -1;
}
+ char *data_filename = argv[1];
+--- a/src/ffindex_from_fasta.c
++++ b/src/ffindex_from_fasta.c
+@@ -31,9 +31,9 @@
-+ index->num_max_entries = num_max_entries;
- index->file = index_file;
- index->index_data = ffindex_mmap_data(index_file, &(index->index_data_size));
- if(index->index_data_size == 0)
+ void usage(char *program_name)
+ {
+- fprintf(stderr, "USAGE: %s -v | [-s] data_filename index_filename fasta_filename\n"
++ fprintf(stderr, "USAGE: ffindex_from_fasta -v | [-s] data_filename index_filename fasta_filename\n"
+ "\t-s\tsort index file\n"
+- "\nDesigned and implemented by Andreas W. Hauser <hauser at genzentrum.lmu.de>.\n", program_name);
++ "\nDesigned and implemented by Andreas W. Hauser <hauser at genzentrum.lmu.de>.\n");
+ }
+
+ int main(int argn, char **argv)
Added: trunk/packages/ffindex/trunk/debian/patches/hardening
===================================================================
--- trunk/packages/ffindex/trunk/debian/patches/hardening (rev 0)
+++ trunk/packages/ffindex/trunk/debian/patches/hardening 2012-06-27 09:52:43 UTC (rev 11483)
@@ -0,0 +1,18 @@
+From: Laszlo Kajan <lkajan at rostlab.org>
+Subject: hardening flags
+Forwarded: yes
+
+--- a/src/Makefile
++++ b/src/Makefile
+@@ -15,6 +15,11 @@
+
+ #-Wl,--version-script=libffindex.map
+
++# hardening flags
++CFLAGS+=-g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security
++CPPFLAGS+=-D_FORTIFY_SOURCE=2
++LDFLAGS+=-Wl,-z,relro
++
+ ifdef HAVE_MPI
+ all: ffindex_apply_mpi ffindex_apply ffindex_from_fasta ffindex_get ffindex_build ffindex_modify libffindex.a libffindex.so
+ else
Added: trunk/packages/ffindex/trunk/debian/patches/num_max_entries_bug
===================================================================
--- trunk/packages/ffindex/trunk/debian/patches/num_max_entries_bug (rev 0)
+++ trunk/packages/ffindex/trunk/debian/patches/num_max_entries_bug 2012-06-27 09:52:43 UTC (rev 11483)
@@ -0,0 +1,23 @@
+From: Laszlo Kajan <lkajan at rostlab.org>
+Subject: index pointer used before checking for NULL
+Forwarded: yes
+
+--- a/src/ffindex.c
++++ b/src/ffindex.c
+@@ -223,13 +223,14 @@
+ num_max_entries = FFINDEX_MAX_INDEX_ENTRIES_DEFAULT;
+ size_t nbytes = sizeof(ffindex_index_t) + (sizeof(ffindex_entry_t) * num_max_entries);
+ ffindex_index_t *index = (ffindex_index_t *)malloc(nbytes);
+- index->num_max_entries = num_max_entries;
+ if(index == NULL)
+ {
+- fferror_print(__FILE__, __LINE__, __func__, "malloc failed");
++ char buf[2048]; snprintf( buf, 2048, "malloc failed to allocate %ld bytes", nbytes );
++ fferror_print(__FILE__, __LINE__, __func__, buf );
+ return NULL;
+ }
+
++ index->num_max_entries = num_max_entries;
+ index->file = index_file;
+ index->index_data = ffindex_mmap_data(index_file, &(index->index_data_size));
+ if(index->index_data_size == 0)
Modified: trunk/packages/ffindex/trunk/debian/patches/series
===================================================================
--- trunk/packages/ffindex/trunk/debian/patches/series 2012-06-27 06:01:53 UTC (rev 11482)
+++ trunk/packages/ffindex/trunk/debian/patches/series 2012-06-27 09:52:43 UTC (rev 11483)
@@ -1 +1,4 @@
Makefile
+hardening
+num_max_entries_bug
+40000000_bug
Modified: trunk/packages/ffindex/trunk/debian/rules
===================================================================
--- trunk/packages/ffindex/trunk/debian/rules 2012-06-27 06:01:53 UTC (rev 11482)
+++ trunk/packages/ffindex/trunk/debian/rules 2012-06-27 09:52:43 UTC (rev 11483)
@@ -3,6 +3,10 @@
# Uncomment this to turn on verbose mode.
export DH_VERBOSE=1
+# [Laszlo Kajan <lkajan at rostlab.org>]
+# ffindex has an overly large default for FFINDEX_MAX_INDEX_ENTRIES (40000000). In order to require less RAM for testing the package (make check), a more modest value is set below.
+export FFINDEX_MAX_INDEX_ENTRIES=20000
+
%:
dh $@ --parallel
More information about the debian-med-commit
mailing list