[med-svn] [ncbi-vdb] 01/01: New upstream version 2.8.2-2+dfsg
Andreas Tille
tille at debian.org
Thu Oct 19 21:46:51 UTC 2017
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to annotated tag upstream/2.8.2-2+dfsg
in repository ncbi-vdb.
commit a68f5ecb2a45ba7dc6dae53198fd57f75025eb97
Author: Andreas Tille <tille at debian.org>
Date: Thu Oct 19 22:48:08 2017 +0200
New upstream version 2.8.2-2+dfsg
---
.gitignore | 1 +
CHANGES.md | 23 +
build/Makefile.env | 12 +-
build/Makefile.install | 91 +-
build/Makefile.vers | 2 +-
interfaces/align/bam.h | 33 +-
interfaces/insdc/sra.vschema | 4 +-
.../libkdb.vers.h => interfaces/kfg/kart-priv.h | 24 +-
interfaces/kfg/kart.h | 40 +-
interfaces/kfs/directory.h | 7 +
interfaces/kfs/filetools.h | 12 +-
libs/ngs/VByteBlob.h => interfaces/kfs/limitfile.h | 34 +-
interfaces/klib/debug.h | 3 +-
interfaces/klib/namelist.h | 7 +
interfaces/klib/time.h | 16 +
interfaces/kns/endpoint.h | 6 +
interfaces/kns/kns-mgr-priv.h | 1 -
interfaces/kns/stream.h | 7 +
interfaces/ktst/unit_test_suite.hpp | 28 +-
interfaces/sra/generic-fastq.vschema | 9 +
interfaces/tui/tui.hpp | 1 +
interfaces/vfs/path.h | 11 +-
interfaces/vfs/resolver.h | 14 +-
.../VByteBlob.h => interfaces/vfs/services-priv.h | 40 +-
interfaces/vfs/services.h | 138 +
libs/align/align-access.c | 55 +-
libs/align/bam.c | 610 +--
libs/ascp/ascp.c | 2 +-
libs/blast/blast-mgr.c | 2 +-
libs/blast/reference.c | 5 +-
libs/kdb/libkdb.vers.h | 2 +-
libs/kfg/config.c | 87 +-
libs/kfg/kart.c | 331 +-
libs/kfg/keystore.c | 6 +
libs/kfs/Makefile | 3 +-
libs/kfs/cacheteefile.c | 129 +-
libs/kfs/from_to_namelist.c | 55 +-
libs/kfs/gzip.c | 110 +-
libs/kfs/limitfile.c | 206 ++
libs/kfs/unix/sysdir.c | 24 +
libs/klib/log.c | 28 +-
libs/klib/release-vers.h | 2 +-
libs/klib/time.c | 28 +
libs/klib/unix/systime.c | 66 +-
libs/klib/vector_namelist.c | 59 +
libs/kns/Makefile | 1 +
libs/kns/http-client.c | 113 +-
libs/kns/http-file.c | 54 +-
libs/kns/http-priv.h | 16 +-
libs/kns/http.c | 1 -
libs/kns/linux/sysendpoint.c | 10 +
libs/kns/manager.c | 110 +-
libs/kns/mgr-priv.h | 7 +
libs/kns/stream-from-buffer.c | 192 +
libs/kns/tls.c | 158 +-
libs/kns/unix/syssock.c | 11 +
libs/ktst/testcase.cpp | 2 +-
libs/ktst/testenv.cpp | 15 +
libs/ncbi-vdb/libncbi-vdb.vers | 2 +-
libs/ngs/BAM_ReadCollection.c | 191 +-
libs/ngs/CSRA1_Alignment.c | 2 +-
libs/ngs/CSRA1_Read.c | 101 +-
libs/ngs/CSRA1_ReadCollection.c | 4 +-
libs/ngs/CSRA1_Reference.c | 33 +-
libs/ngs/CSRA1_Reference.h | 7 +-
libs/ngs/CSRA1_ReferenceWindow.c | 185 +-
libs/ngs/Makefile | 6 +-
libs/ngs/NGS_FragmentBlob.c | 26 +-
libs/ngs/NGS_FragmentBlob.h | 6 +
libs/ngs/NGS_FragmentBlobIterator.c | 6 +
libs/ngs/NGS_ReadCollection.c | 8 +-
libs/ngs/NGS_Reference.c | 127 +-
libs/ngs/NGS_Reference.h | 31 +-
libs/ngs/NGS_ReferenceBlob.c | 296 ++
libs/ngs/NGS_ReferenceBlob.h | 100 +
...tBlobIterator.c => NGS_ReferenceBlobIterator.c} | 76 +-
libs/ngs/NGS_ReferenceBlobIterator.h | 82 +
libs/ngs/SRA_DB_ReadCollection.c | 10 +-
libs/ngs/SRA_Read.c | 1 +
libs/ngs/SRA_Read.h | 5 +-
libs/ngs/SRA_ReadCollection.c | 10 +-
libs/ngs/VByteBlob.c | 31 +-
libs/ngs/VByteBlob.h | 4 +-
libs/tui/tui_dlg_helper.c | 128 +-
libs/vdb/cursor-cmn.c | 2 +-
libs/vdb/libvdb.vers.h | 2 +-
libs/vfs/Makefile | 5 +-
libs/vfs/manager.c | 38 +-
libs/vfs/path-priv.h | 79 +
libs/vfs/path.c | 290 +-
libs/vfs/remote-services.c | 3905 ++++++++++++++++++++
libs/vfs/resolver-priv.h | 7 +-
libs/vfs/resolver.c | 315 +-
libs/vfs/services-priv.h | 126 +
libs/vfs/services.c | 280 ++
libs/vfs/srv-response.c | 580 +++
py_vdb/L1-manager.py | 57 +
py_vdb/L10-fastq.py | 81 +
py_vdb/L2-table_read.py | 80 +
py_vdb/L3-table_write.py | 94 +
py_vdb/L4-database_read.py | 58 +
py_vdb/L5-database_write.py | 111 +
py_vdb/L6-meta_read.py | 47 +
py_vdb/L7-table-rnd-write.py | 49 +
py_vdb/L8-import-csv.py | 44 +
py_vdb/L9-index_usage.py | 57 +
py_vdb/custom_fastq.py | 27 +
py_vdb/ref_var.py | 375 ++
py_vdb/tst_config.py | 48 +
py_vdb/vdb.py | 2229 +++++++++++
setup/konfigure.perl | 99 +-
setup/package.prl | 12 +-
test/Makefile | 1 +
build/Makefile.vers => test/align-access/Makefile | 35 +-
test/align-access/align-access.hpp | 182 +
.../release-vers.h => test/align-access/main.c | 37 +-
test/align-access/test.cpp | 192 +
test/align-access/test.pl | 132 +
test/install/.gitignore | 2 +
test/install/Makefile | 139 +
test/kdb/kdbtest.cpp | 31 +-
test/kfg/flat-sra-kfg.cpp | 4 +-
test/kfg/keystoretest.cpp | 48 +-
test/kfg/kfgtest.cpp | 3 +-
test/kfs/Makefile | 70 +-
test/kfs/cachetee-out-of-space.cpp | 187 +
test/kfs/cacheteetest.cpp | 4 +-
test/kfs/fuse_proxy.c | 668 ++++
test/kfs/kdf.cpp | 303 ++
test/kfs/run-cachetee-out-of-space-test.sh | 87 +
test/klib/Makefile | 13 +
test/klib/test-log.cpp | 36 +
test/klib/test-time.cpp | 87 +
test/kns/Makefile | 98 +-
test/kns/test-proxy-with-scheme.cpp | 162 +
test/kns/test-proxy.cpp | 16 +
.../env-with-schema-and-port/environment | 1 +
.../test-proxy/env-with-schema-and-port/expected | 1 +
test/kns/test-proxy/env-with-schema/environment | 1 +
test/kns/test-proxy/env-with-schema/expected | 1 +
.../kns/test-proxy/kfg-with-schema-and-port/config | 1 +
.../test-proxy/kfg-with-schema-and-port/expected | 1 +
test/kns/test-proxy/kfg-with-schema/config | 1 +
test/kns/test-proxy/kfg-with-schema/expected | 1 +
test/krypto/Makefile | 10 +-
test/ktst/ktsttest.cpp | 52 +
test/ngs-java/ngs_test_CSRA1.java | 292 +-
test/ngs/Makefile | 43 +-
test/ngs/ngs_c_fixture.hpp | 2 +-
test/ngs/ngstest_byteblob.cpp | 314 ++
test/ngs/ngstest_csra1.cpp | 502 ---
test/ngs/ngstest_fragmentblob.cpp | 29 +-
test/ngs/ngstest_reference.cpp | 582 ++-
test/ngs/ngstest_referenceblob.cpp | 773 ++++
test/vdb/Makefile | 22 +-
test/vdb/test-VDB-3060.cpp | 2 +-
test/vdb/test-VDB-3061.cpp | 40 +-
test/vdb/test-VDB-3305.cpp | 161 +
test/vdb/test-vdb.cpp | 90 +-
test/vfs/Makefile | 26 +-
.../redirect-rejected-names-cgi-http-to-https.cpp | 10 +-
test/vfs/resolvertest.cpp | 12 +-
test/vfs/test-caching.cpp | 4 +-
test/vfs/test-names-30.cpp | 503 +++
vdb3/itf/kfc/except.hpp | 36 +-
vdb3/itf/kfc/memmgr.hpp | 9 +-
vdb3/itf/kfc/ptr.hpp | 20 +-
vdb3/itf/kfc/rsrc.hpp | 15 +-
vdb3/itf/kfc/string.hpp | 9 +-
vdb3/src/kfc/Makefile | 16 +-
vdb3/src/kfc/rsrc.cpp | 1 +
vdb3/src/kfc/string.cpp | 80 +-
172 files changed, 17841 insertions(+), 1998 deletions(-)
diff --git a/.gitignore b/.gitignore
index 40b02ab..bdfbf75 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,3 +32,4 @@ user.status
.\#*
core
*.pyc
+.gdb_history
diff --git a/CHANGES.md b/CHANGES.md
index 02cfc3f..74dcdb5 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,28 @@
# NCBI External Developer Release:
+## NCBI VDB 2.8.2
+**March 6, 2017**
+
+ **blast**: Updated blast library to be able to process runs having empty rows
+ **build**: Added ability to specify ncbi-vdb/configure --with-magic-prefix. Look for libraries in (lib lib64) when running "configure --with-...-prefix"
+ **build**: configure detects location of ngs libraries
+ **build**: configure was fixed to skip options unrecognized by gcc 4.4.7
+ **build**: created sra-toolkit Debian package
+ **build**: fixed a bug in 'configure' when in could not find source files in repository saved with non-standard name
+ **build, kfg**: install updates certs.kfg along with other configuration files
+ **build, ncbi-vdb, sra-tools**: installation will back up old configuration files if they differ from the ones being installed
+ **kfs, vdb**: Fixes errors occuring when configuration is missing
+ **klib**: Fixed logging error reported as "log failure: RC(rcText,rcString,rcConverting,rcBuffer,rcInsufficient)"
+ **kns**: SRA tools respect standard set of environment variables for proxy specification
+ **kns**: rewrote socket code to use proper local error code switches, especially Windows
+ **kns**: updated mbedtls library to version 2.4.1
+ **ncbi-vdb**: configure in detects existence of ngs jar file
+ **ncbi-vdb, ngs, ngs-tools, sra-tools**: eliminated memcpy from sources due to potential for overlap
+ **test**: Improved testing framework
+ **vdb**: increased tenacity in flushing data to disk to accommodate behaviors of file systems such as Lustre.
+ **vdb-blast**: Implemented vdb-blast API for retrieving reference sequence
+
+
## NCBI VDB 2.8.1
**December 22, 2016**
diff --git a/build/Makefile.env b/build/Makefile.env
index 17bed71..6d5fa94 100644
--- a/build/Makefile.env
+++ b/build/Makefile.env
@@ -131,7 +131,7 @@ SUBDIRS = bin test-bin ilib lib
OUTDIRS = schema
# compilation rules
-#MAKE_JOBS = 16
+MAKE_JOBS = 1
stdcompile: makedirs
@ $(MAKE_CMD) -j$(MAKE_JOBS) $(TARGDIR)/compile
@@ -191,7 +191,7 @@ endif
.PHONY: compile stdcompile stdclean removelinks makedirs vers-includes rebuild-dirlinks
.PHONY: stdjclean makejdirs
-# configuration targets
+# configuration targets
out:
@ echo $(OUTDIR) > $(TOP)/build/OUTDIR.$(BUILD_OS)
@ $(MAKE) TOP=$(TOP) -f $(TOP)/build/Makefile.env rebuild-dirlinks
@@ -337,11 +337,11 @@ LDFLAGS += $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT)
#-------------------------------------------------------------------------------
# runtests
-#
+#
# MallocScribble=1 is for catching allocation problems on Mac
#
ifeq ($(RUNTESTS_OVERRIDE),)
-runtests: std $(TEST_TOOLS)
+runtests: all $(TEST_TOOLS)
@ export VDB_CONFIG=$(VDB_CONFIG);\
export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;export MallocScribble=1;\
for i in $(TEST_TOOLS);\
@@ -356,11 +356,11 @@ endif
#-------------------------------------------------------------------------------
# slowtests
-#
+#
# $(SLOWTESTSDATADIR) should be used to create temporary test files
SLOWTESTSDATADIR ?= /panfs/pan1.be-md.ncbi.nlm.nih.gov/sra-test/slowtests/$(shell whoami)
-slowtests: std $(SLOW_TEST_TOOLS)
+slowtests: all $(SLOW_TEST_TOOLS)
@ export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;\
for i in $(SLOW_TEST_TOOLS);\
do\
diff --git a/build/Makefile.install b/build/Makefile.install
index 743ff72..7ae10a7 100644
--- a/build/Makefile.install
+++ b/build/Makefile.install
@@ -29,8 +29,28 @@ include $(TOP)/build/Makefile.shell
include $(CONFIG_FILE)
#-------------------------------------------------------------------------------
+# set up installation locations
+
+#fake root for debugging
+#uncomment this line and change the following test for root ( see under install: ) to succeed:
+#ROOT = ~/root
+ifeq (linux, $(OS))
+ ifeq (0, $(shell id -u))
+ LINUX_ROOT = true
+ endif
+endif
+
+ifeq (true, $(LINUX_ROOT))
+ KONFIG_DIR = $(ROOT)/etc/ncbi
+ INCLUDE_SYMLINK = $(ROOT)/usr/include/ncbi-vdb
+ PROFILE_FILE = $(ROOT)/etc/profile.d/ncbi-vdb
+else
+ KONFIG_DIR = $(LIB_TARGET)/ncbi
+endif
+
+#-------------------------------------------------------------------------------
# install
-#
+#
LIBRARIES_TO_INSTALL = \
ncbi-vdb.$(VERSION_LIBX) \
ncbi-vdb.$(VERSION_SHLX) \
@@ -45,7 +65,7 @@ LIBRARIES_WITH_PREFIX = \
checkversion:
-copylibs: checkversion $(LIB_TARGET)
+copylibs: checkversion $(LIB_TARGET)
@ echo "Installing libraries to $(LIB_TARGET)"
@ $(MAKE) -f $(TOP)/build/Makefile.install $(LIBRARIES_WITH_PREFIX)
@@ -54,39 +74,26 @@ copyincludes: checkversion
@mkdir -p $(INST_INCDIR)/ncbi-vdb
@cp $(TOP)/interfaces/ngs/ncbi/NGS.hpp $(INST_INCDIR)/ncbi-vdb
-#fake root for debugging
-#uncomment this line and change the test for root ( see under install: ) to succeed:
-#ROOT = ~/root
-
-INCLUDE_SYMLINK = $(ROOT)/usr/include/ncbi-vdb
-PROFILE_FILE = $(ROOT)/etc/profile.d/ncbi-vdb
-KONFIG_DIR = $(ROOT)/etc/ncbi
-
-ifeq (linux, $(OS))
- ifeq (0, $(shell id -u))
- LINUX_ROOT = true
- endif
-endif
-
-install: copylibs copyincludes
-ifeq (true, $(LINUX_ROOT))
- @ # install symlink to includes
- @ echo "Updating $(INCLUDE_SYMLINK)"
- @ rm -f $(INCLUDE_SYMLINK)
- @ ln -s $(INST_INCDIR)/ncbi-vdb $(INCLUDE_SYMLINK)
+copykfg: checkversion
@ #
@ # install configuration file(s)
@ echo "Installing configuration files to $(KONFIG_DIR)"
@ mkdir -p $(KONFIG_DIR)
- if [ -f $(KONFIG_DIR)/certs.kfg ] ; \
- then mv -v $(KONFIG_DIR)/certs.kfg $(KONFIG_DIR)/certs.kfg.orig ; fi
- if [ -f $(KONFIG_DIR)/default.kfg ] ; \
- then mv -v $(KONFIG_DIR)/default.kfg $(KONFIG_DIR)/default.kfg.orig ; fi
- if [ -f $(KONFIG_DIR)/ncbi-vdb.kfg ] ; \
- then mv -v $(KONFIG_DIR)/ncbi-vdb.kfg $(KONFIG_DIR)/ncbi-vdb.kfg.orig ; fi
+ @ if [ -f $(KONFIG_DIR)/certs.kfg -a "$$(diff $(TOP)/libs/kfg/certs.kfg $(KONFIG_DIR)/certs.kfg 2>/dev/null)" != "" ] ; \
+ then mv -v $(KONFIG_DIR)/certs.kfg $(KONFIG_DIR)/certs.kfg.orig ; fi
@ cp $(TOP)/libs/kfg/certs.kfg $(KONFIG_DIR)
+ @ if [ -f $(KONFIG_DIR)/default.kfg -a "$$(diff $(TOP)/libs/kfg/default.kfg $(KONFIG_DIR)/default.kfg 2>/dev/null)" != "" ] ; \
+ then mv -v $(KONFIG_DIR)/default.kfg $(KONFIG_DIR)/default.kfg.orig ; fi
+ @ if [ -f $(KONFIG_DIR)/ncbi-vdb.kfg ] ; \
+ then mv -v $(KONFIG_DIR)/ncbi-vdb.kfg $(KONFIG_DIR)/ncbi-vdb.kfg.orig ; fi
@ cp $(TOP)/libs/kfg/default.kfg $(KONFIG_DIR)
- @ #
+
+install: copylibs copyincludes copykfg
+ifeq (true, $(LINUX_ROOT))
+ @ # install symlink to includes
+ @ echo "Updating $(INCLUDE_SYMLINK)"
+ @ rm -f $(INCLUDE_SYMLINK)
+ @ ln -s $(INST_INCDIR)/ncbi-vdb $(INCLUDE_SYMLINK)
@ echo "Updating $(PROFILE_FILE).[c]sh"
@ printf \
"#version $(VERSION)\n"\
@@ -105,39 +112,21 @@ ifeq (true, $(LINUX_ROOT))
@ #
@ echo "Use \$$NCBI_VDB_LIBDIR in your link commands, e.g.:"
@ echo " ld -L\$$NCBI_VDB_LIBDIR -lncbi-vdb ..."
-else
- @ #
- @ echo "Installing configuration files to $(LIB_TARGET)/ncbi/"
- @ mkdir -p $(LIB_TARGET)/ncbi
- if [ -f $(LIB_TARGET)/ncbi/certs.kfg ] ; then \
- mv -v $(LIB_TARGET)/ncbi/certs.kfg \
- $(LIB_TARGET)/ncbi/certs.kfg.orig ; fi
- if [ -f $(LIB_TARGET)/ncbi/default.kfg ] ; then \
- mv -v $(LIB_TARGET)/ncbi/default.kfg \
- $(LIB_TARGET)/ncbi/default.kfg.orig ; fi
- if [ -f $(LIB_TARGET)/ncbi/ncbi-vdb.kfg ] ; then \
- mv -v $(LIB_TARGET)/ncbi/ncbi-vdb.kfg \
- $(LIB_TARGET)/ncbi/ncbi-vdb.kfg.orig ; fi
- @ cp $(TOP)/libs/kfg/certs.kfg $(LIB_TARGET)/ncbi
- @ cp $(TOP)/libs/kfg/default.kfg $(LIB_TARGET)/ncbi
@ #
@ echo "Please add $(LIB_TARGET) to your LD_LIBRARY_PATH, e.g.:"
@ echo " export LD_LIBRARY_PATH=$(LIB_TARGET):\$$LD_LIBRARY_PATH"
@ #
@ echo "Use $(LIB_TARGET) in your link commands, e.g.:"
- @ echo " export NCBI_VDB_LIBDIR=$(LIB_TARGET)"
+ @ echo " export NCBI_VDB_LIBDIR=$(LIB_TARGET)"
@ echo " ld -L\$$NCBI_VDB_LIBDIR -lncbi-vdb ..."
endif
#-------------------------------------------------------------------------------
# uninstall
-#
+#
-TO_UNINSTALL = $(LIB_TARGET)/libncbi-vdb.* $(LIB_TARGET)/libncbi-wvdb.* $(LIB_TARGET)/libncbi-ngs-c++.*
-ifneq (true, $(LINUX_ROOT))
- TO_UNINSTALL += $(LIB_TARGET)/ncbi/ncbi-vdb.kfg $(LIB_TARGET)/ncbi/default.kfg
-endif
-TO_UNINSTALL_AS_ROOT = $(INCLUDE_SYMLINK) $(KONFIG_DIR)/ncbi-vdb.kfg $(KONFIG_DIR)/default.kfg $(PROFILE_FILE).sh $(PROFILE_FILE).csh
+TO_UNINSTALL = $(LIB_TARGET)/libncbi-vdb* $(LIB_TARGET)/libncbi-wvdb* $(LIB_TARGET)/libncbi-ngs-c++* $(INST_INCDIR)/ncbi-vdb $(KONFIG_DIR)/*.kfg
+TO_UNINSTALL_AS_ROOT = $(INCLUDE_SYMLINK) $(PROFILE_FILE).sh $(PROFILE_FILE).csh
uninstall:
@ echo "Uninstalling $(TO_UNINSTALL) ..."
diff --git a/build/Makefile.vers b/build/Makefile.vers
index 499fbe8..1ed427e 100644
--- a/build/Makefile.vers
+++ b/build/Makefile.vers
@@ -23,4 +23,4 @@
# ===========================================================================
# NCBI-VDB and library version
-VERSION = 2.8.1
+VERSION = 2.8.2
diff --git a/interfaces/align/bam.h b/interfaces/align/bam.h
index 7081c7b..5987b8c 100644
--- a/interfaces/align/bam.h
+++ b/interfaces/align/bam.h
@@ -57,6 +57,8 @@ struct AlignAccessAlignmentEnumerator;
*/
typedef struct BAMAlignment BAMAlignment;
+typedef struct BAMFileSlice BAMFileSlice;
+
/* GetBAMAlignment
* get property
@@ -73,6 +75,7 @@ ALIGN_EXTERN rc_t CC AlignAccessAlignmentEnumeratorGetBAMAlignment
ALIGN_EXTERN rc_t CC BAMAlignmentAddRef ( const BAMAlignment *self );
ALIGN_EXTERN rc_t CC BAMAlignmentRelease ( const BAMAlignment *self );
+ALIGN_EXTERN uint64_t BAMAlignmentGetFilePos(const BAMAlignment *self);
/* GetReadLength
* get the sequence length
@@ -705,9 +708,33 @@ ALIGN_EXTERN bool CC BAMFileIsIndexed ( const BAMFile *self );
*/
ALIGN_EXTERN bool CC BAMFileIndexHasRefSeqId ( const BAMFile *self, uint32_t refSeqId );
-/* Seek
- * seeks a half-open zero-based interval on a particular reference
- * rcSelf, rcIncomplete
+
+/* MakeSlice
+ * Makes a slice iterator on the BAM file
+ * the result should be free'd
+ */
+ALIGN_EXTERN rc_t CC BAMFileMakeSlice(const BAMFile *self, BAMFileSlice **rslt, uint32_t refSeqId, uint64_t alignStart, uint64_t alignEnd);
+
+
+/* ReadSlice
+ * Read next record as specified by the slice iterator
+ *
+ * Example:
+ * BAMFileSlice *slice;
+ * const BAMAlignment *record;
+ *
+ * BAMFileMakeSlice(file, &slice, 0, 141484029, 141495762);
+ * while (BAMFileReadSlice(file, &record, slice) == 0) {
+ * ... do something ...
+ * BAMAlignmentRelease(record);
+ * }
+ * free(slice);
+ */
+ALIGN_EXTERN rc_t CC BAMFileReadSlice(const BAMFile *self, const BAMAlignment **rslt, BAMFileSlice *slice);
+
+/* Seek ******** DEPRECATED *******
+ *
+ * rcFunction, rcUnsupported
*/
ALIGN_EXTERN rc_t CC BAMFileSeek ( const BAMFile *self, uint32_t refSeqId, uint64_t alignStart, uint64_t alignEnd );
diff --git a/interfaces/insdc/sra.vschema b/interfaces/insdc/sra.vschema
index 5cfbfd1..7a31f93 100644
--- a/interfaces/insdc/sra.vschema
+++ b/interfaces/insdc/sra.vschema
@@ -122,7 +122,7 @@ table INSDC:SRA:tbl:spotcoord #1
* 1.0.1 - split X and Y into spotcoord table
* 1.0.2 - added ability to get name from TRACE_NAME
*/
-table INSDC:SRA:tbl:spotname #1.0.2 = INSDC:SRA:tbl:spotcoord #1
+table INSDC:SRA:tbl:spotname #1.0.1 = INSDC:SRA:tbl:spotcoord #1
{
/* NAME
* external name for spot
@@ -145,7 +145,7 @@ table INSDC:SRA:tbl:spotname #1.0.2 = INSDC:SRA:tbl:spotcoord #1
| INSDC:SRA:format_spot_name ( out_name_fmt, out_x_coord, out_y_coord )
| INSDC:SRA:format_spot_name_no_coord (out_name_fmt)
| out_spot_name
- | out_trace_name
+ // | out_trace_name // temporarily commented out to revert version back to 1.0.1
;
diff --git a/libs/kdb/libkdb.vers.h b/interfaces/kfg/kart-priv.h
similarity index 80%
copy from libs/kdb/libkdb.vers.h
copy to interfaces/kfg/kart-priv.h
index 557fb9c..9c8bc56 100644
--- a/libs/kdb/libkdb.vers.h
+++ b/interfaces/kfg/kart-priv.h
@@ -1,3 +1,7 @@
+#ifndef _h_kfg_kart_priv_
+#define _h_kfg_kart_priv_
+
+
/*===========================================================================
*
* PUBLIC DOMAIN NOTICE
@@ -24,4 +28,22 @@
*
*/
-#define LIBKDB_VERS 0x02070016
+
+#include <kfg/kart.h>
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+LIB_EXPORT rc_t CC KartMake2 ( Kart ** kart );
+LIB_EXPORT rc_t CC KartAddRow ( Kart * self, const char * row, size_t size );
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _h_kfg_kart_priv_ */
diff --git a/interfaces/kfg/kart.h b/interfaces/kfg/kart.h
index fa54fd4..b9aa1d5 100644
--- a/interfaces/kfg/kart.h
+++ b/interfaces/kfg/kart.h
@@ -43,7 +43,30 @@
extern "C" {
#endif
+
+/* EObjectType
+ * Extensible controlled vocabulary object type.
+ * The list is defined in Described in Cart File Format 2.0 */
+typedef enum {
+ eOT_undefined,
+ eOT_empty,
+ eOT_dbgap,
+ eOT_provisional,
+ eOT_srapub,
+ eOT_sragap,
+ eOT_srapub_source,
+ eOT_sragap_source,
+ eOT_srapub_files,
+ eOT_sragap_files,
+ eOT_refseq,
+ eOT_wgs,
+ eOT_na,
+ eOT_nakmer,
+} EObjectType;
+
+
struct KDirectory;
+struct KSrvError;
/* AA-833 */
@@ -54,8 +77,8 @@ KFG_EXTERN rc_t CC KartItemRelease(const KartItem *self);
/** Do not release the returned String !
* N.B. returned String is not required to be NULL-terminated !
-KFG_EXTERN rc_t CC KartItemTypeId(const KartItem *self, const String **elem);
*/
+/* VERSION 1.0 ****************************************************************/
KFG_EXTERN rc_t CC KartItemProjId(const KartItem *self, const String **elem);
KFG_EXTERN rc_t CC KartItemProjIdNumber(const KartItem *self, uint64_t *id);
KFG_EXTERN rc_t CC KartItemItemId(const KartItem *self, const String **elem);
@@ -63,6 +86,18 @@ KFG_EXTERN rc_t CC KartItemItemIdNumber(const KartItem *self, uint64_t *id);
KFG_EXTERN rc_t CC KartItemAccession(const KartItem *self, const String **elem);
KFG_EXTERN rc_t CC KartItemName(const KartItem *self, const String **elem);
KFG_EXTERN rc_t CC KartItemItemDesc(const KartItem *self, const String **elem);
+/* VERSION 2.0 ****************************************************************/
+KFG_EXTERN rc_t CC KartItemObjType (const KartItem *self, const String **elem );
+KFG_EXTERN rc_t CC KartItemPath (const KartItem *self, const String **elem );
+KFG_EXTERN rc_t CC KartItemSize (const KartItem *self, const String **elem );
+
+/* Kart object can be created:
+ * - from a kart file,
+ * - or as result to search service call.
+ * In the latter it can contain errors.
+ */
+KFG_EXTERN rc_t CC KartItemGetError (const KartItem *self,
+ const struct KSrvError ** error );
typedef struct Kart Kart;
@@ -79,10 +114,11 @@ KFG_EXTERN rc_t CC KartMakeText(const struct KDirectory *dir, const char *path,
KFG_EXTERN rc_t CC KartPrint(const Kart *self);
KFG_EXTERN rc_t CC KartPrintNumbered(const Kart *self);
-KFG_EXTERN rc_t CC KartMakeNextItem(Kart *self, const KartItem **item);
+KFG_EXTERN rc_t CC KartMakeNextItem(const Kart *self, const KartItem **item);
KFG_EXTERN rc_t CC KartItemsProcessed(const Kart *self, uint16_t *number);
+
#ifdef __cplusplus
}
#endif
diff --git a/interfaces/kfs/directory.h b/interfaces/kfs/directory.h
index ba69175..087e677 100644
--- a/interfaces/kfs/directory.h
+++ b/interfaces/kfs/directory.h
@@ -527,6 +527,9 @@ KFS_EXTERN rc_t CC KDirectoryCopyPaths_v1 ( const KDirectory_v1 * src_dir,
KFS_EXTERN rc_t CC KDirectoryCopy_v1 ( const KDirectory_v1 *src_dir,
KDirectory_v1 *dst_dir, bool recursive, const char *src, const char *dst );
+KFS_EXTERN rc_t CC KDirectoryGetDiskFreeSpace_v1 ( const KDirectory * self,
+ uint64_t * free_bytes_available, uint64_t * total_number_of_bytes );
+
/* NativeDir
* returns a native file-system directory node reference
* the directory root will be "/" and set to the native
@@ -581,6 +584,10 @@ typedef struct NAME_VERS ( KDirectory, KDIRECTORY_VERS ) KDirectory;
#define KDirectoryCopyPath NAME_VERS ( KDirectoryCopyPath , KDIRECTORY_VERS )
#define KDirectoryCopyPaths NAME_VERS ( KDirectoryCopyPaths , KDIRECTORY_VERS )
#define KDirectoryCopy NAME_VERS ( KDirectoryCopy , KDIRECTORY_VERS )
+
+#define KDirectoryGetDiskFreeSpace NAME_VERS \
+ ( KDirectoryGetDiskFreeSpace , KDIRECTORY_VERS )
+
#define KDirectoryNativeDir NAME_VERS ( KDirectoryNativeDir , KDIRECTORY_VERS )
diff --git a/interfaces/kfs/filetools.h b/interfaces/kfs/filetools.h
index ff9eb6f..3f1eca5 100644
--- a/interfaces/kfs/filetools.h
+++ b/interfaces/kfs/filetools.h
@@ -35,6 +35,10 @@
#include <kfs/file.h>
#endif
+#ifndef _h_kfs_directory_
+#include <kfs/directory.h>
+#endif
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -78,7 +82,13 @@ KFS_EXTERN rc_t CC WriteNameListToKFile( struct KFile * self, const VNamelist *
KFS_EXTERN rc_t CC WriteNamelistToFileByName( const VNamelist * namelist,
const char * filename, const char * delim );
-
+/* ReadDirEntriesIntoToNamelist
+ * creates a VNamelist-instance, iterates over the entries of the given KDirectory,
+ * and enters them into the created Namelist ( sorted on request )
+ */
+KFS_EXTERN rc_t CC ReadDirEntriesIntoToNamelist( VNamelist ** namelist, const KDirectory * dir,
+ bool perform_sort, bool add_files, bool add_dirs, const char * path );
+
#ifdef __cplusplus
}
#endif
diff --git a/libs/ngs/VByteBlob.h b/interfaces/kfs/limitfile.h
similarity index 62%
copy from libs/ngs/VByteBlob.h
copy to interfaces/kfs/limitfile.h
index e9d623b..9e1c887 100644
--- a/libs/ngs/VByteBlob.h
+++ b/interfaces/kfs/limitfile.h
@@ -24,24 +24,40 @@
*
*/
-#ifndef _h_vbyteblob_
-#define _h_vbyteblob_
+#ifndef _h_kfs_limitfile_
+#define _h_kfs_limitfile_
-#include <kfc/ctx.h>
+#ifndef _h_kfs_extern_
+#include <kfs/extern.h>
+#endif
-struct VBlob;
+#ifndef _h_klib_defs_
+#include <klib/defs.h>
+#endif
#ifdef __cplusplus
extern "C" {
#endif
-/* Calculate the biggest available contiguous data portion of the blob:
-* starts at rowId, ends before a repeated value or at the end of the blob
-*/
-void VByteBlob_ContiguousChunk ( const struct VBlob* blob, ctx_t ctx, int64_t rowId, const void** data, uint64_t* size, bool stopAtRepeat );
+struct KFile;
+
+/*--------------------------------------------------------------------------
+ * MakeLimitFile
+ * a file that limits reads or writes to being within a particular block
+ *
+ * "obj" [ OUT ] - return parameter for newly created limit file
+ *
+ * "original" [ IN ] - file to be wrapped.
+ * on success, will have a new reference created to it.
+ *
+ * "block_size" [ IN ] - the size of the emulated I/O block. MUST be
+ * an even power of two, or the function will fail.
+ */
+KFS_EXTERN rc_t CC KFileMakeLimitFile ( struct KFile ** obj,
+ struct KFile const * original, size_t block_size );
#ifdef __cplusplus
}
#endif
-#endif /* _h_vbyteblob_ */
+#endif /* _h_kfs_limitfile_ */
diff --git a/interfaces/klib/debug.h b/interfaces/klib/debug.h
index 40b1633..97c68d7 100644
--- a/interfaces/klib/debug.h
+++ b/interfaces/klib/debug.h
@@ -149,7 +149,8 @@ extern "C" {
_condition(KFS,POS) _condition(KFS,PAGE)
#define KNS_CONDITIONS() \
- _condition(KNS,ERR) _condition(KNS,HTTP) _condition(KNS,MGR) _condition(KNS,SOCKET)
+ _condition(KNS,DNS) _condition(KNS,ERR) _condition(KNS,HTTP) \
+ _condition(KNS,MGR) _condition(KNS,SOCKET) _condition(KNS,TLS)
#define VFS_CONDITIONS() \
_condition(VFS,MGR) _condition(VFS,PATH) _condition(VFS,SERVICE)
diff --git a/interfaces/klib/namelist.h b/interfaces/klib/namelist.h
index 8439d72..9569188 100644
--- a/interfaces/klib/namelist.h
+++ b/interfaces/klib/namelist.h
@@ -195,7 +195,14 @@ KLIB_EXTERN rc_t CC VNamelistSplitString ( VNamelist * list,
KLIB_EXTERN rc_t CC VNamelistSplitStr ( VNamelist * list,
const char * str, const uint32_t delim );
+/* creates a VNamelist from a KNamelist
+ */
+KLIB_EXTERN rc_t CC VNamelistFromKNamelist ( VNamelist ** list, const KNamelist * src );
+/* creates a copy of a VNamelist
+ */
+KLIB_EXTERN rc_t CC CopyVNamelist ( VNamelist ** list, const VNamelist * src );
+
/* VNamelistFromString, VNamelistFromStr
* splits a String or char-ptr and creates VNamelist from parts
*/
diff --git a/interfaces/klib/time.h b/interfaces/klib/time.h
index 98989f6..50b3ed5 100644
--- a/interfaces/klib/time.h
+++ b/interfaces/klib/time.h
@@ -83,6 +83,22 @@ KLIB_EXTERN const KTime* CC KTimeLocal ( KTime *kt, KTime_t ts );
KLIB_EXTERN const KTime* CC KTimeGlobal ( KTime *kt, KTime_t ts );
+/* Iso8601
+ * populate "s" from "ts" according to ISO-8601:
+ * YYYY-MM-DDThh:mm:ssTZD
+ */
+KLIB_EXTERN size_t CC KTimeIso8601 ( KTime_t ts, char * s, size_t size );
+
+
+/* FromIso8601
+ * populate "kt" from "s" accoring to ISO-8601:
+ * YYYY-MM-DDThh:mm:ssTZD
+ * or YYYY-MM-DDThh:mm:ss
+ */
+KLIB_EXTERN const KTime* CC KTimeFromIso8601 ( KTime *kt, const char * s,
+ size_t size );
+
+
/* MakeTime
* make a KTime_t from KTime
*/
diff --git a/interfaces/kns/endpoint.h b/interfaces/kns/endpoint.h
index 0e4c29c..ed91c11 100644
--- a/interfaces/kns/endpoint.h
+++ b/interfaces/kns/endpoint.h
@@ -30,6 +30,10 @@
#include <kns/extern.h>
#endif
+#ifndef _h_klib_defs_
+#include <klib/defs.h> /* rc_t */
+#endif
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -78,6 +82,8 @@ struct KEndPoint
char ipc_name [ IPC_NAME_MAX ];
} u;
+ char ip_address [ 256 ];
+
KEndPointType type;
};
diff --git a/interfaces/kns/kns-mgr-priv.h b/interfaces/kns/kns-mgr-priv.h
index c6404ff..7a5eea0 100644
--- a/interfaces/kns/kns-mgr-priv.h
+++ b/interfaces/kns/kns-mgr-priv.h
@@ -141,7 +141,6 @@ struct URLBlock
extern void URLBlockInit ( URLBlock *self );
extern rc_t ParseUrl ( URLBlock * b, const char * url, size_t url_size );
-
#ifdef __cplusplus
}
#endif
diff --git a/interfaces/kns/stream.h b/interfaces/kns/stream.h
index 7935767..f6ff71a 100644
--- a/interfaces/kns/stream.h
+++ b/interfaces/kns/stream.h
@@ -215,6 +215,13 @@ KNS_EXTERN rc_t CC KStreamMakeBuffered ( KStream ** buffered,
const KStream * in, KStream * out, size_t bufer_size );
+/* MakeFromBuffer
+ * makes "stream" from provided "buffer" of "size" bytes
+ */
+KNS_EXTERN rc_t CC KStreamMakeFromBuffer ( KStream ** stream,
+ const char * buffer, size_t size );
+
+
#ifdef __cplusplus
}
#endif
diff --git a/interfaces/ktst/unit_test_suite.hpp b/interfaces/ktst/unit_test_suite.hpp
index 88e686e..672103a 100644
--- a/interfaces/ktst/unit_test_suite.hpp
+++ b/interfaces/ktst/unit_test_suite.hpp
@@ -168,15 +168,15 @@ private:
#endif
};
+
class TestCase {
- void Init(const char* name);
+ void Init(const std::string& name);
public:
typedef void ( TestCase ::* TestMethod ) ();
protected:
- TestCase(const std::string &name) { Init(name.c_str()); }
- TestCase(const char* name) { Init(name); }
+ TestCase(const std::string &name) { Init(name); }
public:
// explicit destruction, to be used before calling exit() in out-of-process test runner
@@ -184,7 +184,7 @@ public:
public:
ncbi::NK::counter_t GetErrorCounter(void) { return _ec; }
- const char* GetName(void) const { return _name; }
+ const char * GetName(void) const { return _name . c_str (); }
void ErrorCounterAdd(ncbi::NK::counter_t ec) { _ec += ec; }
protected:
@@ -434,10 +434,28 @@ protected:
#define REQUIRE_NOT_NULL(e1) AssertNotNull((e1), #e1, __FILE__,__LINE__, true)
private:
- const char* _name;
+ std::string _name;
ncbi::NK::counter_t _ec;
};
+
+class SharedTest : protected TestCase {
+ TestCase * _dad;
+
+protected:
+ SharedTest ( TestCase * dad, const char * name )
+ : TestCase ( std::string ( dad -> GetName () ) + "." + name )
+ , _dad ( dad )
+ {
+ assert ( _dad );
+ }
+
+ ~SharedTest ( void ) {
+ _dad -> ErrorCounterAdd ( GetErrorCounter () );
+ }
+};
+
+
class TestInvoker {
protected:
TestInvoker(const std::string& name) : _name(name), _ec(0) {}
diff --git a/interfaces/sra/generic-fastq.vschema b/interfaces/sra/generic-fastq.vschema
index 2c29d13..66d1b76 100644
--- a/interfaces/sra/generic-fastq.vschema
+++ b/interfaces/sra/generic-fastq.vschema
@@ -32,6 +32,7 @@ version 1;
include 'insdc/sra.vschema';
include 'ncbi/sra.vschema';
include 'ncbi/clip.vschema';
+include 'sra/illumina.vschema';
include 'ncbi/spotname.vschema';
/* tokenize_spot_name - currently ascii only capability */
@@ -81,6 +82,14 @@ database NCBI:SRA:GenericFastq:db #1
};
/*--------------------------------------------------------------------------
+ * Illumina db defined based on sra/illumina.vschema
+ */
+database NCBI:SRA:Illumina:db #1
+{
+ table NCBI:SRA:Illumina:tbl:phred:v2 #1.0.4 SEQUENCE;
+};
+
+/*--------------------------------------------------------------------------
* NCBI:SRA:GenericFastq:sequence_no_name
* Generic Fastq SRA Platform (without name)
*/
diff --git a/interfaces/tui/tui.hpp b/interfaces/tui/tui.hpp
index 71a596c..1af48ce 100644
--- a/interfaces/tui/tui.hpp
+++ b/interfaces/tui/tui.hpp
@@ -372,6 +372,7 @@ class Dlg
bool IsChanged( void ) { return KTUIDlgGetChanged ( dlg_ ); };
bool GetRect( Tui_Rect &r ) { return ( KTUIDlgGetRect ( dlg_, &( r.r_ ) ) == 0 ); };
+ Tui_Rect GetRect( void ) { Tui_Rect r; GetRect( r ); return r; }
bool SetRect( Tui_Rect const &r, bool redraw ) { return ( KTUIDlgSetRect ( dlg_, &( r.r_ ), redraw ) == 0 ); };
virtual bool Resize( Tui_Rect const &r );
diff --git a/interfaces/vfs/path.h b/interfaces/vfs/path.h
index 33f9c11..ab5e0b2 100644
--- a/interfaces/vfs/path.h
+++ b/interfaces/vfs/path.h
@@ -268,7 +268,7 @@ VFS_EXTERN rc_t CC VPathMakeSysPath ( const VPath * self,
VFS_EXTERN rc_t CC VPathMakeString ( const VPath * self,
struct String const ** str );
-
+
/* Get*
* retrieves internal parts
* returns pointers to internal String data
@@ -286,6 +286,15 @@ VFS_EXTERN rc_t CC VPathGetFragment ( const VPath * self, struct String * str );
/* TEMPORARY */
VFS_EXTERN uint32_t CC VPathGetOid ( const VPath * self );
+/* The following parts are set
+ when VPath was created from name resolver response */
+/* GetId: retrieve object-id returned by name resolver */
+VFS_EXTERN rc_t CC VPathGetId ( const VPath * self, struct String * str );
+VFS_EXTERN rc_t CC VPathGetTicket ( const VPath * self, struct String * str );
+VFS_EXTERN KTime_t CC VPathGetModDate ( const VPath * self );
+VFS_EXTERN size_t CC VPathGetSize ( const VPath * self );
+VFS_EXTERN const uint8_t * CC VPathGetMd5 ( const VPath * self );
+
#ifdef __cplusplus
}
diff --git a/interfaces/vfs/resolver.h b/interfaces/vfs/resolver.h
index 4611165..c7f596f 100644
--- a/interfaces/vfs/resolver.h
+++ b/interfaces/vfs/resolver.h
@@ -84,19 +84,24 @@ enum
{
/* version 1.1 protocols */
eProtocolNone = 0
+ , eProtocolDefault = eProtocolNone
, eProtocolHttp = 1
, eProtocolFasp = 2
/* version 1.2 protocols */
, eProtocolHttps = 3
- /* values 3..7 are available for future */
+ /* version 3.0 protocols */
+ , eProtocolFile = 4
+ , eProtocolS3 = 5
+
+ /* values 6..7 are available for future */
, eProtocolLast
, eProtocolMax = eProtocolLast - 1
, eProtocolMask = 7
- , eProtocolMaxPref = 3
+ , eProtocolMaxPref = 5
/* macros for building multi-protocol constants
ordered by preference from least to most significant bits */
@@ -108,6 +113,10 @@ enum
( VRemoteProtocolsMake2 ( p1, p2 ) | \
( ( ( VRemoteProtocols ) ( p3 ) & eProtocolMask ) << ( 3 * 2 ) ) )
+#define VRemoteProtocolsMake4( p1, p2, p3, p4 ) \
+ ( VRemoteProtocolsMake3 ( p1, p2, p3 ) | \
+ ( ( ( VRemoteProtocols ) ( p4 ) & eProtocolMask ) << ( 3 * 3 ) ) )
+
, eProtocolFaspHttp = VRemoteProtocolsMake2 ( eProtocolFasp, eProtocolHttp )
, eProtocolHttpFasp = VRemoteProtocolsMake2 ( eProtocolHttp, eProtocolFasp )
, eProtocolHttpsHttp = VRemoteProtocolsMake2 ( eProtocolHttps, eProtocolHttp )
@@ -120,6 +129,7 @@ enum
, eProtocolHttpHttpsFasp = VRemoteProtocolsMake3 ( eProtocolHttp, eProtocolHttps, eProtocolFasp )
, eProtocolHttpsFaspHttp = VRemoteProtocolsMake3 ( eProtocolHttps, eProtocolFasp, eProtocolHttp )
, eProtocolHttpsHttpFasp = VRemoteProtocolsMake3 ( eProtocolHttps, eProtocolHttp, eProtocolFasp )
+ , eProtocolFileFaspHttpHttps= VRemoteProtocolsMake4 ( eProtocolFile, eProtocolFasp, eProtocolHttp, eProtocolHttps )
};
/* Parse
diff --git a/libs/ngs/VByteBlob.h b/interfaces/vfs/services-priv.h
similarity index 57%
copy from libs/ngs/VByteBlob.h
copy to interfaces/vfs/services-priv.h
index e9d623b..1470d1d 100644
--- a/libs/ngs/VByteBlob.h
+++ b/interfaces/vfs/services-priv.h
@@ -1,3 +1,6 @@
+#ifndef _h_vfs_services_priv_
+#define _h_vfs_services_priv_
+
/*===========================================================================
*
* PUBLIC DOMAIN NOTICE
@@ -24,24 +27,43 @@
*
*/
-#ifndef _h_vbyteblob_
-#define _h_vbyteblob_
-#include <kfc/ctx.h>
+#include <vfs/services.h> /* KService */
-struct VBlob;
#ifdef __cplusplus
extern "C" {
#endif
-/* Calculate the biggest available contiguous data portion of the blob:
-* starts at rowId, ends before a repeated value or at the end of the blob
-*/
-void VByteBlob_ContiguousChunk ( const struct VBlob* blob, ctx_t ctx, int64_t rowId, const void** data, uint64_t* size, bool stopAtRepeat );
+
+struct KNSManager;
+
+
+rc_t KServiceNamesQueryExt ( KService * self, VRemoteProtocols protocols,
+ const char * cgi, const char * version,
+ const KSrvResponse ** response );
+
+rc_t KServiceNamesExecuteExt ( KService * self, VRemoteProtocols protocols,
+ const char * cgi, const char * version,
+ const struct KSrvResponse ** result );
+
+rc_t KServiceSearchExecuteExt ( KService * self,
+ const char * cgi, const char * version,
+ const struct Kart ** result );
+
+
+rc_t KServiceTestNamesExecuteExt ( KService * self, VRemoteProtocols protocols,
+ const char * cgi, const char * version,
+ const struct KSrvResponse ** result, const char * expected );
+
+
+rc_t KService1Search ( const struct KNSManager * mgr, const char * cgi,
+ const char * acc, const struct Kart ** result );
+
#ifdef __cplusplus
}
#endif
-#endif /* _h_vbyteblob_ */
+
+#endif /* _h_vfs_services_priv_ */
diff --git a/interfaces/vfs/services.h b/interfaces/vfs/services.h
new file mode 100644
index 0000000..d047fe2
--- /dev/null
+++ b/interfaces/vfs/services.h
@@ -0,0 +1,138 @@
+#ifndef _h_vfs_services_
+#define _h_vfs_services_
+
+
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+
+#include <kfg/kart.h> /* EObjectType */
+#include <vfs/resolver.h> /* VRemoteProtocols */
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+struct Kart;
+struct KNSManager;
+
+typedef struct KService KService;
+typedef struct KSrvError KSrvError;
+typedef struct KSrvResponse KSrvResponse;
+
+
+/******************************************************************************/
+/* KService - EXTERNAL Service */
+
+/* Make KService object */
+rc_t KServiceMake ( KService ** self );
+
+/* Release KService object */
+rc_t KServiceRelease ( KService * self );
+
+/* Add an Id ( Accession or Object-Id ) to service request */
+rc_t KServiceAddId ( KService * self, const char * id );
+
+/* Add a dbGaP Project to service request */
+rc_t KServiceAddProject ( KService * self, uint32_t id );
+
+
+/************************** name service - version 3 **************************/
+/* Execute Names Service Call using current default protocol version;
+ * get KSrvResponse - it contains:
+ * - remote/local/cache location for every requested Id
+ * - or KSrvError
+ */
+rc_t KServiceNamesQuery ( KService * self, VRemoteProtocols protocols,
+ const KSrvResponse ** response );
+
+/************************** search service - version 1 ************************/
+/* Execute Search Service Call; get Kart response */
+rc_t KServiceSearchExecute ( KService * self,
+ const struct Kart ** response );
+
+
+/************************** KSrvResponse **************************/
+/* Release:
+ * Release KSrvResponse object */
+rc_t KSrvResponseRelease ( const KSrvResponse * self );
+
+/* Length:
+ * Number of elements in KSrvResponse */
+uint32_t KSrvResponseLength ( const KSrvResponse * self );
+
+/* GetPath:
+ * Get KSrvResponse element number "idx" for "protocol":
+ * - remote "path"/"vdbcache",
+ * or "error"
+ */
+rc_t KSrvResponseGetPath ( const KSrvResponse * self, uint32_t idx,
+ VRemoteProtocols p, const struct VPath ** path,
+ const struct VPath ** vdbcache, const KSrvError ** error );
+
+/* GetLocal:
+ * get local path
+ */
+rc_t KSrvResponseGetLocal ( const KSrvResponse * self, uint32_t idx,
+ const struct VPath ** local );
+
+/* GetCache:
+ * get cache path
+ */
+rc_t KSrvResponseGetCache ( const KSrvResponse * self, uint32_t idx,
+ const struct VPath ** cache );
+
+/************************** KSrvError ******************************
+ * KSrvError is generated for Id-s from request that produced an error response
+ */
+rc_t KSrvErrorRelease ( const KSrvError * self );
+rc_t KSrvErrorAddRef ( const KSrvError * self );
+
+/* Rc - rc code corresponding to this Error */
+rc_t KSrvErrorRc ( const KSrvError * self, rc_t * rc );
+
+/* Code - Status-Code returned by server */
+rc_t KSrvErrorCode ( const KSrvError * self, uint32_t * code );
+
+/* returns pointers to internal String data
+ * Strings remain valid while "self" is valid
+ */
+/* Message - message returned by server */
+rc_t KSrvErrorMessage ( const KSrvError * self, String * message );
+/* Object - Object-Id/Object-Type that produced this Error */
+rc_t KSrvErrorObject ( const KSrvError * self,
+ String * id, EObjectType * type );
+/******************************************************************************/
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _h_vfs_services_ */
diff --git a/libs/align/align-access.c b/libs/align/align-access.c
index 350d007..fe3c2fb 100644
--- a/libs/align/align-access.c
+++ b/libs/align/align-access.c
@@ -300,9 +300,9 @@ LIB_EXPORT rc_t CC AlignAccessRefSeqEnumeratorNext(const AlignAccessRefSeqEnumer
struct AlignAccessAlignmentEnumerator {
const AlignAccessDB *parent;
const BAMAlignment *innerSelf;
+ BAMFileSlice *slice;
uint64_t endpos;
uint64_t startpos;
- BAMFilePosition pos;
atomic32_t refcount;
int atend;
int refSeqID;
@@ -341,16 +341,18 @@ LIB_EXPORT rc_t CC AlignAccessDBEnumerateAlignments(const AlignAccessDB *self, A
return AlignAccessAlignmentEnumeratorNext(*align_enum);
}
-LIB_EXPORT rc_t CC AlignAccessDBWindowedAlignments(
- const AlignAccessDB *self,
- AlignAccessAlignmentEnumerator **align_enum,
- const char *refSeqName, uint64_t pos, uint64_t wsize
-) {
+LIB_EXPORT
+rc_t CC AlignAccessDBWindowedAlignments(const AlignAccessDB *self,
+ AlignAccessAlignmentEnumerator **align_enum,
+ const char *refSeqName, uint64_t pos, uint64_t wsize
+ )
+{
AlignAccessAlignmentEnumerator *lhs;
unsigned i, n;
const BAMRefSeq *rs;
uint64_t endpos = pos + wsize;
rc_t rc;
+ BAMFileSlice *slice;
*align_enum = NULL;
@@ -364,10 +366,12 @@ LIB_EXPORT rc_t CC AlignAccessDBWindowedAlignments(
return RC(rcAlign, rcTable, rcConstructing, rcParam, rcInvalid);
}
- if (wsize == 0 || endpos > rs->length)
+ if (endpos > rs->length || wsize == 0)
endpos = rs->length;
- rc = BAMFileSeek(self->innerSelf, i, pos, endpos);
+ rc = BAMFileMakeSlice(self->innerSelf, &slice, i, pos, endpos);
+ if (rc == 0 && slice == NULL)
+ return RC(rcAlign, rcTable, rcConstructing, rcMemory, rcExhausted);
if ( rc != 0 )
{
if ( GetRCState( rc ) == rcNotFound && GetRCObject( rc ) == (enum RCObject)rcData )
@@ -381,6 +385,7 @@ LIB_EXPORT rc_t CC AlignAccessDBWindowedAlignments(
lhs->refSeqID = i;
lhs->endpos = endpos;
lhs->startpos = pos;
+ lhs->slice = slice;
*align_enum = lhs;
return AlignAccessAlignmentEnumeratorNext(*align_enum);
@@ -399,8 +404,17 @@ AGAIN:
if (self->atend != 0)
return AlignAccessAlignmentEnumeratorEOFCode;
- BAMFileGetPosition(self->parent->innerSelf, &self->pos);
- rc = BAMFileRead2(self->parent->innerSelf, &self->innerSelf);
+ if (self->slice == NULL) {
+ rc = BAMFileRead2(self->parent->innerSelf, &self->innerSelf);
+ if (rc) {
+ if (GetRCState(rc) == rcNotFound && GetRCObject(rc) == rcRow) {
+ self->atend = 1;
+ rc = AlignAccessAlignmentEnumeratorEOFCode;
+ }
+ }
+ return rc;
+ }
+ rc = BAMFileReadSlice(self->parent->innerSelf, &self->innerSelf, self->slice);
if (rc) {
if (GetRCState(rc) == rcNotFound && GetRCObject(rc) == rcRow) {
self->atend = 1;
@@ -408,9 +422,9 @@ AGAIN:
}
return rc;
}
- if (self->refSeqID == -1)
- return 0;
-
+ if (!BAMAlignmentIsMapped(self->innerSelf))
+ goto AGAIN;
+
BAMAlignmentGetRefSeqId(self->innerSelf, &refSeqID);
if (self->refSeqID != refSeqID) {
self->atend = 1;
@@ -419,15 +433,14 @@ AGAIN:
else if (self->endpos != 0) {
int64_t pos;
uint32_t length;
- uint64_t endpos;
-
+
BAMAlignmentGetPosition2(self->innerSelf, &pos, &length);
- if (pos < 0 || pos >= (int64_t)self->endpos) {
+ if (pos >= (int64_t)self->endpos) {
self->atend = 1;
rc = AlignAccessAlignmentEnumeratorEOFCode;
}
else {
- endpos = (uint64_t)pos + length;
+ int64_t const endpos = pos + length;
if (endpos <= self->startpos)
goto AGAIN;
}
@@ -443,6 +456,7 @@ LIB_EXPORT rc_t CC AlignAccessAlignmentEnumeratorAddRef ( const AlignAccessAlign
static
rc_t CC AlignAccessAlignmentEnumeratorWhack(AlignAccessAlignmentEnumerator *self) {
+ free(self->slice);
if (self->innerSelf)
BAMAlignmentRelease(self->innerSelf);
AlignAccessDBRelease(self->parent);
@@ -801,7 +815,7 @@ LIB_EXPORT rc_t CC AlignAccessAlignmentEnumeratorGetRecordID(const AlignAccessAl
if (self == NULL)
return 0;
- *(BAMFilePosition *)result = self->pos;
+ *(BAMFilePosition *)result = BAMAlignmentGetFilePos(self->innerSelf);
return 0;
}
@@ -810,3 +824,8 @@ LIB_EXPORT rc_t CC AlignAccessAlignmentEnumeratorGetBAMAlignment(const AlignAcce
*result = self->innerSelf;
return BAMAlignmentAddRef(self->innerSelf);
}
+
+LIB_EXPORT rc_t CC AlignAccessAlignmentEnumeratorGetSAM(const AlignAccessAlignmentEnumerator *self, size_t *const actSize, size_t const maxsize, char *const buffer)
+{
+ return BAMAlignmentFormatSAM(self->innerSelf, actSize, maxsize, buffer);
+}
diff --git a/libs/align/bam.c b/libs/align/bam.c
index 678d0af..9656ed0 100644
--- a/libs/align/bam.c
+++ b/libs/align/bam.c
@@ -653,9 +653,29 @@ static rc_t BGZThreadFileInit(BGZThreadFile *self, const KFile *kfp, BGZFile_vt
#endif
/* MARK: BAMFile structures */
+#define MAX_BIN 37449
+
+typedef struct BAMFileRange {
+ BAMFilePosition start;
+ BAMFilePosition end;
+} BAMFileRange;
+
+typedef struct BAMIndexReference {
+ int intervals;
+ int binSize[MAX_BIN];
+ int binStart[MAX_BIN];
+ BAMFilePosition start;
+ BAMFilePosition end;
+ BAMFileRange *bin;
+ BAMFilePosition *interval;
+} BAMIndexReference;
struct BAMIndex {
- BAMFilePosition *refSeq[1];
+ int numRefs;
+ BAMFileRange *rng;
+ BAMFilePosition *pos;
+ void *data; // allocated
+ BAMIndexReference ref[1];
};
struct BAMFile {
@@ -731,6 +751,7 @@ struct offset_size_s {
struct BAMAlignment {
KRefcount refcount;
+ BAMFilePosition pos;
BAMFile *parent;
bam_alignment const *data;
uint8_t *storage;
@@ -812,7 +833,7 @@ static void const *getCigarBase(BAMAlignment const *cself)
return &cself->data->raw[cself->cigar];
}
-static int opt_tag_cmp(uint8_t const a[2], uint8_t const b[2])
+static int opt_tag_cmp(char const a[2], char const b[2])
{
int const d0 = (int)a[0] - (int)b[0];
return d0 ? d0 : ((int)a[1] - (int)b[1]);
@@ -823,8 +844,8 @@ static int64_t CC OptTag_sort(void const *A, void const *B, void *ctx)
BAMAlignment const *const self = ctx;
unsigned const a_off = ((struct offset_size_s const *)A)->offset;
unsigned const b_off = ((struct offset_size_s const *)B)->offset;
- uint8_t const *const a = &self->data->raw[a_off];
- uint8_t const *const b = &self->data->raw[b_off];
+ char const *const a = (char const *)&self->data->raw[a_off];
+ char const *const b = (char const *)&self->data->raw[b_off];
int const diff = opt_tag_cmp(a, b);
return diff ? (int64_t)diff : (int64_t)a - (int64_t)b;
@@ -837,7 +858,7 @@ static unsigned tag_findfirst(BAMAlignment const *const self, char const tag[2])
while (f < e) {
unsigned const m = (f + e) >> 1;
- char const *const mtag = &self->data->raw[self->extra[m].offset];
+ char const *const mtag = (char const *)&self->data->raw[self->extra[m].offset];
int const d = opt_tag_cmp(tag, mtag);
if (d > 0)
@@ -855,7 +876,7 @@ static unsigned tag_runlength(BAMAlignment const *const self,
unsigned n;
for (n = 0; n + at < self->numExtra; ++n) {
- if (opt_tag_cmp(tag, &self->data->raw[self->extra[n + at].offset]) != 0)
+ if (opt_tag_cmp(tag, (char const *)&self->data->raw[self->extra[n + at].offset]) != 0)
break;
}
return n;
@@ -1839,8 +1860,12 @@ LIB_EXPORT float CC BAMFileGetProportionalPosition(const BAMFile *self)
return self->vt.FileProPos(&self->file);
}
+static BAMFilePosition BAMFileGetPositionInt(const BAMFile *self) {
+ return (self->fpos_cur << 16) | self->bufCurrent;
+}
+
LIB_EXPORT rc_t CC BAMFileGetPosition(const BAMFile *self, BAMFilePosition *pos) {
- *pos = (self->fpos_cur << 16) | self->bufCurrent;
+ *pos = BAMFileGetPositionInt(self);
return 0;
}
@@ -2112,9 +2137,11 @@ static unsigned BAMAlignmentSetOffsets(BAMAlignment *const self)
}
static bool BAMAlignmentInit(BAMAlignment *const self, unsigned const maxsize,
+ BAMFilePosition pos,
unsigned const datasize, void const *const data)
{
memset(self, 0, sizeof(*self));
+ self->pos = pos;
self->data = data;
self->datasize = datasize;
{
@@ -2135,9 +2162,11 @@ static bool BAMAlignmentInit(BAMAlignment *const self, unsigned const maxsize,
}
static bool BAMAlignmentInitLog(BAMAlignment *const self, unsigned const maxsize,
+ BAMFilePosition pos,
unsigned const datasize, void const *const data)
{
memset(self, 0, sizeof(*self));
+ self->pos = pos;
self->data = data;
self->datasize = datasize;
{
@@ -2230,6 +2259,7 @@ rc_t BAMFileReadNoCopy(BAMFile *const self, unsigned actsize[], BAMAlignment rhs
if (maxPeek < 4)
return SILENT_RC(rcAlign, rcFile, rcReading, rcBuffer, rcNotAvailable);
else {
+ BAMFilePosition curPos = (self->fpos_cur << 16) | self->bufCurrent;
int32_t const i32 = BAMFilePeekI32(self);
if (i32 <= 0)
@@ -2238,7 +2268,7 @@ rc_t BAMFileReadNoCopy(BAMFile *const self, unsigned actsize[], BAMAlignment rhs
if (maxPeek < ( uint32_t ) i32 + 4)
return SILENT_RC(rcAlign, rcFile, rcReading, rcBuffer, rcNotAvailable);
- isgood = BAMAlignmentInitLog(rhs, maxsize, i32, BAMFilePeek(self, 4));
+ isgood = BAMAlignmentInitLog(rhs, maxsize, curPos, i32, BAMFilePeek(self, 4));
rhs[0].parent = self;
KRefcountInit(&rhs->refcount, 1, "BAMAlignment", "ReadNoCopy", "");
}
@@ -2255,7 +2285,7 @@ unsigned BAMAlignmentSizeFromData(unsigned const datasize, void const *data)
{
BAMAlignment temp;
- BAMAlignmentInit(&temp, sizeof(temp), datasize, data);
+ BAMAlignmentInit(&temp, sizeof(temp), 0, datasize, data);
return BAMAlignmentSize(temp.numExtra);
}
@@ -2280,6 +2310,7 @@ rc_t BAMFileReadCopy(BAMFile *const self, BAMAlignment const *rslt[], bool const
void const *data;
unsigned datasize;
rc_t rc;
+ BAMFilePosition curPos = (self->fpos_cur << 16) | self->bufCurrent;
rslt[0] = NULL;
{
@@ -2318,7 +2349,7 @@ rc_t BAMFileReadCopy(BAMFile *const self, BAMAlignment const *rslt[], bool const
BAMAlignment *const y = malloc(rsltsize);
if (y) {
- if ((log ? BAMAlignmentInitLog : BAMAlignmentInit)(y, rsltsize, datasize, data)) {
+ if ((log ? BAMAlignmentInitLog : BAMAlignmentInit)(y, rsltsize, curPos, datasize, data)) {
if (storage == NULL)
self->bufLocker = y;
else
@@ -2541,6 +2572,11 @@ LIB_EXPORT uint16_t CC BAMAlignmentBadFields(const BAMAlignment *self)
/* MARK: BAM Alignment accessors */
+LIB_EXPORT BAMFilePosition BAMAlignmentGetFilePos(const BAMAlignment *self)
+{
+ return self->pos;
+}
+
static uint32_t BAMAlignmentGetCigarElement(const BAMAlignment *self, unsigned i)
{
return LE2HUI32(&((uint8_t const *)getCigarBase(self))[i * 4]);
@@ -2560,7 +2596,7 @@ LIB_EXPORT rc_t CC BAMAlignmentGetPosition(const BAMAlignment *cself, int64_t *r
LIB_EXPORT bool CC BAMAlignmentIsMapped(const BAMAlignment *cself)
{
- if (((getFlags(cself) & BAMFlags_SelfIsUnmapped) == 0) && getRefSeqId(cself) >= 0 && getPosition(cself) >= 0)
+ if (((getFlags(cself) & BAMFlags_SelfIsUnmapped) == 0) && getRefSeqId(cself) >= 0 && getPosition(cself) >= 0 && getCigarCount(cself) > 0)
return true;
return false;
}
@@ -3823,7 +3859,7 @@ static uint64_t get_pos(uint8_t const buf[])
return LE2HUI64(buf);
}
-#define MAX_BIN 37449
+#if 0
static uint16_t bin2ival(uint16_t bin)
{
if (bin < 1)
@@ -3869,10 +3905,11 @@ static uint16_t bin_ival_count(uint16_t bin)
return 0;
}
+#endif
enum BAMIndexStructureTypes {
- bai_StartStopPairs,
- bai_16kIntervals
+ bai_pairs,
+ bai_intervals
};
typedef rc_t (*WalkIndexStructureCallBack)(const uint8_t data[], size_t dlen,
@@ -3939,7 +3976,8 @@ rc_t WalkIndexStructure(uint8_t const buf[], size_t const blen,
if (cp + 16 * chunks > blen)
return RC(rcAlign, rcIndex, rcReading, rcData, rcInsufficient);
- rc = func(&buf[cp], 16 * chunks, i, nrefs, bai_StartStopPairs, binNo, bins, chunks, ctx);
+ if (chunks > 0)
+ rc = func(&buf[cp], 16 * chunks, i, nrefs, bai_pairs, binNo, bins, chunks, ctx);
if (rc)
return rc;
cp += 16 * chunks;
@@ -3952,7 +3990,7 @@ rc_t WalkIndexStructure(uint8_t const buf[], size_t const blen,
if (cp + 8 * intervals > blen)
return RC(rcAlign, rcIndex, rcReading, rcData, rcInsufficient);
- rc = func(&buf[cp], 8 * intervals, i, nrefs, bai_16kIntervals, ~(unsigned)0, bins, intervals, ctx);
+ rc = func(&buf[cp], 8 * intervals, i, nrefs, bai_intervals, ~(unsigned)0, bins, intervals, ctx);
if (rc)
return rc;
cp += 8 * intervals;
@@ -3962,148 +4000,35 @@ rc_t WalkIndexStructure(uint8_t const buf[], size_t const blen,
return 0;
}
-struct LoadIndex1_s {
- const BAMFile *self;
- int refNo;
+struct MeasureIndex_s {
unsigned refs;
+ unsigned pairs;
unsigned intervals;
- unsigned total_interval_count;
};
-static
-rc_t LoadIndex1(const uint8_t data[], size_t dlen, unsigned refNo,
- unsigned refs, enum BAMIndexStructureTypes type,
- unsigned binNo, unsigned bins,
- unsigned elements, void *Ctx)
-{
- struct LoadIndex1_s *ctx = (struct LoadIndex1_s *)Ctx;
-
- ctx->refs = refs;
- if (refNo != ctx->refNo) {
- ctx->total_interval_count += ctx->intervals;
- ctx->intervals = 0;
- ctx->refNo = refNo;
- }
- if (elements != 0) {
- if (refNo > ctx->self->refSeqs)
- return RC(rcAlign, rcIndex, rcReading, rcData, rcInvalid);
- ctx->intervals = (ctx->self->refSeq[refNo].length + 16383) >> 14;
- if (type == bai_16kIntervals && elements > ctx->intervals)
- return RC(rcAlign, rcIndex, rcReading, rcData, rcExcessive);
- if (type == bai_StartStopPairs && bin2ival(binNo) > ctx->intervals)
- return RC(rcAlign, rcIndex, rcReading, rcData, rcExcessive);
- }
- return 0;
-}
-
struct LoadIndex2_s {
- const BAMFile *self;
- BAMFilePosition **refSeq;
- BAMFilePosition *cur;
-#if _DEBUGGING
- BAMFilePosition *end;
-#endif
- const uint8_t *base;
- unsigned bins[MAX_BIN + 1];
- bool hasData;
+ BAMIndex *self;
+ unsigned cur_rng;
+ unsigned cur_pos;
};
static
-rc_t LoadIndex2a(const uint8_t data[], size_t dlen, unsigned refNo,
- unsigned refs, enum BAMIndexStructureTypes type,
- unsigned binNo, unsigned bins,
- unsigned elements, struct LoadIndex2_s *ctx)
+rc_t MeasureIndex(const uint8_t data[], size_t dlen, unsigned refNo,
+ unsigned refs, enum BAMIndexStructureTypes type,
+ unsigned binNo, unsigned bins,
+ unsigned elements, void *Ctx)
{
- const unsigned max_ival = (ctx->self->refSeq[refNo].length + 16383) >> 14;
- unsigned i;
- unsigned cp;
- unsigned k;
- uint32_t chunk_count;
- uint64_t minOffset[1u << 15];
-
- assert(ctx->refSeq[refNo] == NULL);
- ctx->refSeq[refNo] = ctx->cur;
- ctx->cur += max_ival;
+ struct MeasureIndex_s *ctx = (struct MeasureIndex_s *)Ctx;
-#if _DEBUGGING
- assert(refNo < ctx->self->refSeqs);
- assert(ctx->cur <= ctx->end);
- assert(elements <= max_ival);
-#endif
- /* get the positions of the first records in the 16kbp intervals */
- for (cp = i = 0; i != elements; ++i, cp += 8)
- ctx->refSeq[refNo][i] = get_pos(&data[cp]);
- /* get the positions of the first records in the 16kbp bins */
- for (i = MAX_BIN; i != 0; ) {
- const unsigned ival = bin2ival(--i);
- const unsigned n_ival = bin_ival_count(i);
- uint64_t found;
-
- cp = ctx->bins[i];
- if (cp == 0)
- continue;
- if (n_ival > 1)
- break;
-
- assert(i == LE2HI32(ctx->base + cp));
- cp += 4;
- chunk_count = LE2HI32(ctx->base + cp); cp += 4;
- found = ctx->refSeq[refNo][ival];
- for (k = 0; k < chunk_count; ++k) {
- const uint64_t start = get_pos(ctx->base + cp);
-
- cp += 16;
- if (found == 0 || start < found)
- found = start;
+ ctx->refs = refs;
+ if (elements != 0) {
+ if (type == bai_intervals) {
+ ctx->intervals += elements;
}
- ctx->refSeq[refNo][ival] = found;
- }
- /* The interval list now contains the offsets to the first alignment
- * that starts at or after the interval's starting position.
- * An interval's starting position is 16kpb * interval number.
- *
- * We will now use the information from the bigger bins to find the
- * offsets of the first chunk of alignments that ends after an
- * interval's first alignment.
- */
- memset(minOffset, 0, sizeof(minOffset));
- for (i = 0; i != MAX_BIN; ++i) {
- const unsigned ival = bin2ival(i);
- unsigned const n_ival = bin_ival_count(i);
-
- cp = ctx->bins[i];
- if (cp == 0)
- continue;
- if (n_ival <= 1)
- break;
-
- chunk_count = LE2HI32(ctx->base + cp + 4); cp += 8;
- for (k = 0; k < chunk_count; ++k) {
- const uint64_t start = get_pos(ctx->base + cp);
- const uint64_t end = get_pos(ctx->base + cp + 8);
- unsigned l;
-
- cp += 16;
- for (l = 0; ival + l < max_ival; ++l) {
- if (start < ctx->refSeq[refNo][ival + l] &&
- ctx->refSeq[refNo][ival + l] <= end &&
- (start < minOffset[ival + l] ||
- minOffset[ival + l] == 0
- )
- )
- {
- minOffset[ival + l] = start;
- }
- }
+ else if (type == bai_pairs && binNo < MAX_BIN) {
+ ctx->pairs += elements;
}
}
- /* update the intervals to the new earlier offsets if any */
- for (i = 0; i != max_ival; ++i) {
- if (minOffset[i] != 0)
- ctx->refSeq[refNo][i] = minOffset[i];
- }
- memset(ctx->bins, 0, sizeof(ctx->bins));
- ctx->hasData = false;
return 0;
}
@@ -4113,67 +4038,74 @@ rc_t LoadIndex2(const uint8_t data[], size_t dlen, unsigned refNo,
unsigned binNo, unsigned bins,
unsigned elements, void *Ctx)
{
- struct LoadIndex2_s *ctx = (struct LoadIndex2_s *)Ctx;
+ struct LoadIndex2_s *const ctx = (struct LoadIndex2_s *)Ctx;
+ BAMIndexReference *const self = &ctx->self->ref[refNo];
+ unsigned i;
- if (type == bai_StartStopPairs) {
- if (binNo < MAX_BIN && elements != 0) {
- ctx->bins[binNo] = &data[-8] - ctx->base;
- ctx->hasData = true;
+ if (type == bai_pairs) {
+ if (binNo < MAX_BIN) {
+ BAMFileRange *const dst = ctx->self->rng + ctx->cur_rng;
+ if (self->bin == NULL)
+ self->bin = dst;
+ self->binSize[binNo] = elements;
+ self->binStart[binNo] = dst - self->bin;
+ for (i = 0; i < elements; ++i) {
+ dst[i].start = get_pos(data + 16 * i + 0);
+ dst[i].end = get_pos(data + 16 * i + 8);
+ }
+ ctx->cur_rng += elements;
+ }
+ else if (binNo == MAX_BIN) {
+ self->start = get_pos(data + 0);
+ self->end = get_pos(data + 8);
}
}
- else if (elements != 0 || ctx->hasData)
- return LoadIndex2a(data, dlen, refNo, refs, type, binNo, bins,
- elements, (struct LoadIndex2_s *)Ctx);
+ else if (type == bai_intervals) {
+ BAMFilePosition *const dst = ctx->self->pos + ctx->cur_pos;
+ self->intervals = elements;
+ self->interval = dst;
+ for (i = 0; i < elements; ++i) {
+ dst[i] = get_pos(data + 8 * i);
+ }
+ ctx->cur_pos += elements;
+ }
return 0;
}
static
rc_t LoadIndex(BAMFile *self, const uint8_t buf[], size_t blen)
{
- BAMIndex *idx;
rc_t rc;
- struct LoadIndex1_s loadIndex1ctx;
- unsigned const posArray = ((uintptr_t)&((const BAMFilePosition **)(NULL))[self->refSeqs]) / sizeof(BAMFilePosition *);
+ struct MeasureIndex_s ctx1;
- memset(&loadIndex1ctx, 0, sizeof(loadIndex1ctx));
- loadIndex1ctx.refNo = -1;
- loadIndex1ctx.self = self;
-
- rc = WalkIndexStructure(buf, blen, LoadIndex1, &loadIndex1ctx);
+ memset(&ctx1, 0, sizeof(ctx1));
+ rc = WalkIndexStructure(buf, blen, MeasureIndex, &ctx1);
if (rc == 0) {
- loadIndex1ctx.total_interval_count += loadIndex1ctx.intervals;
- idx = calloc(1, posArray * sizeof(BAMFilePosition *) +
- loadIndex1ctx.total_interval_count * sizeof(BAMFilePosition));
- if (idx == NULL)
- rc = RC(rcAlign, rcIndex, rcReading, rcMemory, rcExhausted);
- else {
- struct LoadIndex2_s *loadIndex2ctx;
-
- if (self->ndx)
- BAMIndexWhack(self->ndx);
- self->ndx = idx;
-
- loadIndex2ctx = malloc(sizeof(*loadIndex2ctx));
- if (loadIndex2ctx == NULL) {
- rc = RC(rcAlign, rcIndex, rcReading, rcMemory, rcExhausted);
- free(idx);
- self->ndx = NULL;
- }
- else {
- memset(loadIndex2ctx->bins, 0, sizeof(loadIndex2ctx->bins));
- loadIndex2ctx->self = self;
- loadIndex2ctx->refSeq = &idx->refSeq[0];
- loadIndex2ctx->base = buf;
- loadIndex2ctx->hasData = false;
- loadIndex2ctx->cur = (BAMFilePosition *)&idx->refSeq[posArray];
-#if _DEBUGGING
- loadIndex2ctx->end = loadIndex2ctx->cur + loadIndex1ctx.total_interval_count;
-#endif
+ size_t indexRootSize = sizeof(BAMIndex) + sizeof(BAMIndexReference) * (ctx1.refs - 1);
+ size_t indexDataCount = ctx1.pairs * 2 + ctx1.intervals;
+ BAMIndex *idx = calloc(1, indexRootSize);
+ if (idx) {
+ idx->numRefs = ctx1.refs;
+ idx->data = calloc(indexDataCount, sizeof(BAMFilePosition));
+ idx->rng = idx->data;
+ idx->pos = (void *)(idx->rng + ctx1.pairs);
+ if (idx->data) {
+ struct LoadIndex2_s ctx2;
- WalkIndexStructure(buf, blen, LoadIndex2, loadIndex2ctx);
- free(loadIndex2ctx);
+ ctx2.self = idx;
+ ctx2.cur_pos = 0;
+ ctx2.cur_rng = 0;
+ WalkIndexStructure(buf, blen, LoadIndex2, &ctx2);
+
+ if (self->ndx)
+ BAMIndexWhack(self->ndx);
+ self->ndx = idx;
}
+ else
+ rc = RC(rcAlign, rcIndex, rcReading, rcMemory, rcExhausted);
}
+ else
+ rc = RC(rcAlign, rcIndex, rcReading, rcMemory, rcExhausted);
}
return rc;
}
@@ -4258,146 +4190,226 @@ LIB_EXPORT bool CC BAMFileIsIndexed(const BAMFile *self)
LIB_EXPORT bool CC BAMFileIndexHasRefSeqId(const BAMFile *self, uint32_t refSeqId)
{
- if (self && self->ndx && self->ndx->refSeq[refSeqId])
+ if (self && self->ndx && refSeqId < self->ndx->numRefs)
return true;
return false;
}
-static void BAMAlignmentAlignInfo(BAMAlignment *const self,
- int32_t ref[],
- int32_t beg[],
- int32_t end[])
+struct BAMFileSlice {
+ unsigned refSeqId;
+ unsigned sliceStart;
+ unsigned sliceEnd;
+ unsigned ranges;
+ unsigned current;
+ unsigned started;
+ struct BAMFileRange range[1 /* ranges */];
+};
+
+typedef struct BinRange {
+ uint16_t beg, end;
+} BinRange;
+
+typedef struct BinList {
+ BinRange range[6];
+} BinList;
+
+BinList calcBinList(unsigned const refBeg, unsigned const refEnd)
{
- (void)BAMAlignmentSetOffsets(self);
+ BinList rslt;
+ unsigned size = 1 << 29;
+ unsigned offset = 0;
+ unsigned i;
- ref[0] = getRefSeqId(self);
- end[0] = (beg[0] = getPosition(self)) + ReferenceLengthFromCIGAR(self);
+ for (i = 0; i < 6; ++i) {
+ rslt.range[i].beg = offset + refBeg / size;
+ rslt.range[i].end = offset + (refEnd - 1) / size;
+ offset += 1 << (3 * i);
+ size >>= 3;
+ }
+ return rslt;
}
-static
-rc_t BAMFileGetAlignPosAtFilePos(BAMFile *const self,
- BAMFilePosition const *const fpos,
- int32_t ref[],
- int32_t beg[],
- int32_t end[])
+static int64_t CC BAMFileRange_cmp(void const *const A, void const *const B, void *ctx)
+{
+ struct BAMFileRange const *const a = (struct BAMFileRange const *)A;
+ struct BAMFileRange const *const b = (struct BAMFileRange const *)B;
+ return a->start < b->start ? -1 : b->start < a->start ? 1 : a->end < b->end ? -1 : b->end < a->end ? 1 : 0;
+}
+
+#define USE_MAXPOS 0
+#define USE_MINPOS 1
+#define SLICE_VERBOSE 0
+
+static int includeRange(BAMFileRange const *const range, BAMFilePosition const minPos, BAMFilePosition const maxPos)
{
- rc_t rc = BAMFileSetPosition(self, fpos);
+#if USE_MAXPOS
+ if (maxPos != 0 && range->start >= maxPos)
+ return 0;
+#endif
+#if USE_MINPOS
+ if (minPos != 0 && range->end <= minPos)
+ return 0;
+#endif
+ return 1;
+}
+
+static void copyRanges(BAMFileSlice *slice,
+ BAMIndexReference const *const refIndex,
+ BAMFilePosition const minPos,
+ BAMFilePosition const maxPos,
+ BinRange const ranges[6])
+{
+ unsigned j = 0;
+ unsigned i;
- if (rc == 0) {
- BAMAlignment x;
- int32_t i32;
-
- rc = BAMFileReadI32(self, &i32); if (rc) return rc;
- if (i32 <= 0)
- return RC(rcAlign, rcFile, rcReading, rcData, rcInvalid);
+ for (i = 0; i < 6; ++i) {
+ BinRange const r = ranges[i];
+ uint16_t bin = r.beg;
+ while (bin <= r.end) {
+ unsigned const binSize = refIndex->binSize[bin];
+ BAMFileRange const *const range = refIndex->bin + refIndex->binStart[bin];
+ unsigned k;
+
+ for (k = 0; k < binSize; ++k) {
+ if (includeRange(&range[k], minPos, maxPos)) {
+ slice->range[j] = range[k];
+ ++j;
+ }
+ }
+ ++bin;
+ }
+ }
+ assert(j == slice->ranges);
+ ksort(slice->range, slice->ranges, sizeof(slice->range[0]), BAMFileRange_cmp, NULL);
+}
- memset(&x, 0, sizeof(x));
- x.datasize = i32;
- if (x.datasize <= BAMFileMaxPeek(self)) {
- x.data = (void *)&self->buffer[self->bufCurrent];
- BAMFileAdvance(self, x.datasize);
+static unsigned countRanges(BAMIndexReference const *const refIndex,
+ BAMFilePosition const minPos,
+ BAMFilePosition const maxPos,
+ BinRange const ranges[6])
+{
+ unsigned j = 0;
+ unsigned i;
+
+#if SLICE_VERBOSE
+ fprintf(stderr, "min: %016llX; max: %016llX\n", (unsigned long long)minPos, (unsigned long long)maxPos);
+#endif
- BAMAlignmentAlignInfo(&x, ref, beg, end);
+ for (i = 0; i < 6; ++i) {
+ BinRange const r = ranges[i];
+ uint16_t bin = r.beg;
+ while (bin <= r.end) {
+ unsigned const binSize = refIndex->binSize[bin];
+ BAMFileRange const *const range = refIndex->bin + refIndex->binStart[bin];
+ unsigned k;
+
+ for (k = 0; k < binSize; ++k) {
+ if (includeRange(&range[k], minPos, maxPos)) {
+ ++j;
+ }
+ }
+ ++bin;
}
- else {
- void *const temp = malloc(x.datasize);
+ }
+ return j;
+}
+
+static BAMFileSlice *makeSlice(BAMFile const *const self,
+ unsigned const refSeqId,
+ unsigned const alignStart,
+ unsigned const alignEnd)
+{
+ BinList const bins = calcBinList(alignStart, alignEnd);
+ BAMIndexReference const *const refIndex = &self->ndx->ref[refSeqId];
+ unsigned const startBin = alignStart >> 14;
+ unsigned const endBin = ((alignEnd - 1) >> 14) + 1;
+ BAMFilePosition const minPos = refIndex->interval[startBin];
+ BAMFilePosition const maxPos = endBin < refIndex->intervals ? refIndex->interval[endBin] : refIndex->end;
+ unsigned const ranges = countRanges(refIndex, minPos, maxPos, bins.range);
+ BAMFileSlice *slice;
+
+ if (ranges > 0) {
+ slice = malloc(sizeof(*slice) + (ranges - 1) * sizeof(slice->range[0]));
+ if (slice) {
+ slice->refSeqId = refSeqId;
+ slice->sliceStart = alignStart;
+ slice->sliceEnd = alignEnd;
+ slice->ranges = ranges;
+ slice->current = 0;
+ slice->started = 0;
- if (temp) {
- x.data = temp;
-
- rc = BAMFileReadn(self, x.datasize, temp);
- if (rc == 0)
- BAMAlignmentAlignInfo(&x, ref, beg, end);
-
- free(temp);
- }
- else
- rc = RC(rcAlign, rcFile, rcReading, rcMemory, rcExhausted);
+ copyRanges(slice, refIndex, minPos, maxPos, bins.range);
}
}
- return rc;
+ else {
+ slice = calloc(1, sizeof(*slice));
+ if (slice) {
+ slice->refSeqId = refSeqId;
+ slice->sliceStart = alignStart;
+ slice->sliceEnd = alignEnd;
+ }
+ }
+ return slice;
}
-LIB_EXPORT rc_t CC BAMFileSeek(const BAMFile *self, uint32_t refSeqId, uint64_t alignStart, uint64_t alignEnd)
+LIB_EXPORT rc_t CC BAMFileMakeSlice(const BAMFile *self, BAMFileSlice **rslt, uint32_t refSeqId, uint64_t alignStart, uint64_t alignEnd)
{
- BAMFilePosition rpos = 0;
- rc_t rc;
- int32_t prev_alignPos;
- int32_t alignPos;
- int32_t alignEndPos;
- int32_t refSeq;
-
if (self->ndx == NULL)
return RC(rcAlign, rcFile, rcPositioning, rcIndex, rcNotFound);
if (refSeqId >= self->refSeqs)
return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
- if (self->ndx->refSeq[refSeqId] == NULL)
+ if (self->ndx->numRefs <= refSeqId)
return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
if (alignStart >= self->refSeq[refSeqId].length)
return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
+
if (alignEnd > self->refSeq[refSeqId].length)
alignEnd = self->refSeq[refSeqId].length;
+
+ *rslt = makeSlice(self, refSeqId, alignStart, alignEnd);
+ return 0;
+}
+
+LIB_EXPORT rc_t CC BAMFileReadSlice(const BAMFile *cself, const BAMAlignment **rhs, BAMFileSlice *slice)
+{
+ assert(cself != NULL);
+ assert(rhs != NULL);
+ assert(slice != NULL);
+ if (cself == NULL || rhs == NULL || slice == NULL)
+ return RC(rcAlign, rcFile, rcReading, rcParam, rcNull);
- {
- unsigned adjust = 0;
- uint32_t ival_start = (uint32_t)(alignStart >> 14);
+ while (slice->current < slice->ranges) {
+ if (slice->started == 0) {
+ rc_t rc = BAMFileSetPosition(cself, &slice->range[slice->current].start);
+ if (rc) break;
+ }
+ ++slice->started;
{
- uint32_t const ival_end = (uint32_t)((alignEnd + 16383) >> 14);
-
- /* find the first interval >= alignStart that has an alignment */
- while (ival_start != ival_end && (rpos = self->ndx->refSeq[refSeqId][ival_start]) == 0)
- ++ival_start;
+ BAMFilePosition const curPos = BAMFileGetPositionInt(cself);
+ if (curPos < slice->range[slice->current].end) {
+ return BAMFileRead2(cself, rhs);
+ }
+#if SLICE_VERBOSE
+ fprintf(stderr, "slice #%u (%016llX-%016llX) contained %u records\n", slice->current + 1, (unsigned long long)slice->range[slice->current].start, (unsigned long long)slice->range[slice->current].end, slice->started);
+#endif
+ ++slice->current;
+ if (slice->current == slice->ranges)
+ break;
+ if (curPos < slice->range[slice->current].start)
+ slice->started = 0;
}
- if (rpos == 0)
- return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
- do {
- rc = BAMFileGetAlignPosAtFilePos((BAMFile *)self, &rpos, &refSeq, &alignPos, &alignEndPos);
- if (rc)
- return RC(rcAlign, rcFile, rcPositioning, rcIndex, rcInvalid);
- if (refSeq != refSeqId)
- return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
- if (alignPos <= alignEnd)
- break; /* we found the interval we were looking for */
-
- /* we over-shot */
- if (++adjust >= ival_start)
- return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
- if ((rpos = self->ndx->refSeq[refSeqId][ival_start - adjust]) == 0)
- return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
- } while (1);
}
- prev_alignPos = alignPos;
-
- do {
- if (alignPos > alignEnd)
- return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
-
- /* if the alignment overlaps the target range then we are done */
- if (alignPos >= alignStart || alignEndPos >= alignStart)
- return BAMFileSetPosition(self, &rpos);
-
- /* start linear scan */
- BAMFileGetPosition(self, &rpos);
- rc = BAMFileGetAlignPosAtFilePos((BAMFile *)self, &rpos, &refSeq, &alignPos, &alignEndPos);
- if ((int)GetRCObject(rc) == rcData && (int)GetRCState(rc) == rcInsufficient)
- return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
- if (rc) return rc;
- if (refSeq != refSeqId)
- return RC(rcAlign, rcFile, rcPositioning, rcData, rcNotFound);
-
- /* indexed BAM must be sorted by position
- * so verify that we are not out of order
- * whether this means that the index is bad
- * or the file is bad, likely both
- * fix the file and regenerate the index
- */
- if (prev_alignPos > alignPos)
- return RC(rcAlign, rcFile, rcPositioning, rcIndex, rcInvalid);
- prev_alignPos = alignPos;
- } while (1);
+ return RC(rcAlign, rcFile, rcReading, rcRow, rcNotFound);
+}
+
+LIB_EXPORT rc_t CC BAMFileSeek(const BAMFile *self, uint32_t refSeqId, uint64_t alignStart, uint64_t alignEnd)
+{
+ /* Use MakeSlice and ReadSlice instead */
+ return RC(rcAlign, rcFile, rcPositioning, rcFunction, rcUnsupported);
}
static rc_t BAMIndexWhack(const BAMIndex *cself) {
+ free(cself->data);
free((void *)cself);
return 0;
}
@@ -4549,11 +4561,11 @@ rc_t BAMValidateLoadIndex(const uint8_t data[], size_t dlen,
void *Ctx)
{
BAMValidate_ctx_t *ctx = Ctx;
- unsigned const n = type == bai_16kIntervals ? elements : elements * 2;
+ unsigned const n = type == bai_intervals ? elements : elements * 2;
unsigned i;
unsigned j;
- if (type == bai_StartStopPairs && binNo >= MAX_BIN)
+ if (type == bai_pairs && binNo >= MAX_BIN)
return 0;
if (ctx->npositions + elements > ctx->mpositions) {
@@ -4568,7 +4580,7 @@ rc_t BAMValidateLoadIndex(const uint8_t data[], size_t dlen,
for (j = i = 0; i != n; ++i) {
uint64_t const pos = get_pos(&data[i * 8]);
- if (type == bai_StartStopPairs && (i & 1) != 0)
+ if (type == bai_pairs && (i & 1) != 0)
continue;
if (pos) {
diff --git a/libs/ascp/ascp.c b/libs/ascp/ascp.c
index dbf568a..e5bf77e 100644
--- a/libs/ascp/ascp.c
+++ b/libs/ascp/ascp.c
@@ -263,7 +263,7 @@ rc_t ascpParse(const char *buf, size_t len, const char *filename,
{
bool failure = false;
const char *p = buf;
- int64_t l = len;
+ size_t l = len;
assert(buf && len && filename && state && line);
StringInit(line, NULL, 0, 0);
while (true) {
diff --git a/libs/blast/blast-mgr.c b/libs/blast/blast-mgr.c
index 12084a1..1e373ad 100644
--- a/libs/blast/blast-mgr.c
+++ b/libs/blast/blast-mgr.c
@@ -50,7 +50,7 @@
#include <stdio.h> /* fprintf */
#include <string.h> /* memset */
-#define TOOLKIT "sratoolkit2_8_1"
+#define TOOLKIT "sratoolkit2_8_2"
/******************************************************************************/
diff --git a/libs/blast/reference.c b/libs/blast/reference.c
index a6d94be..830c0ef 100644
--- a/libs/blast/reference.c
+++ b/libs/blast/reference.c
@@ -49,7 +49,7 @@
#define MAX_BIT64 (~((uint64_t)-1 >> 1))
static bool _is_set_read_id_reference_bit(uint64_t read_id) {
- return read_id & MAX_BIT64;
+ return ( read_id & MAX_BIT64 ) == 0 ? false : true;
}
static
@@ -185,7 +185,8 @@ static VdbBlastStatus _VdbBlastRefSetCounts(VdbBlastRef *self, uint64_t cur_row,
return eVdbBlastErr;
}
else {
- self->base_count = (self->count - 1) * MAX_SEQ_LEN + read_len;
+ self->base_count
+ = ( size_t ) ( (self->count - 1) * MAX_SEQ_LEN + read_len );
}
}
diff --git a/libs/kdb/libkdb.vers.h b/libs/kdb/libkdb.vers.h
index 557fb9c..ed79692 100644
--- a/libs/kdb/libkdb.vers.h
+++ b/libs/kdb/libkdb.vers.h
@@ -24,4 +24,4 @@
*
*/
-#define LIBKDB_VERS 0x02070016
+#define LIBKDB_VERS 0x02070017
diff --git a/libs/kfg/config.c b/libs/kfg/config.c
index 42dd65a..57efd64 100644
--- a/libs/kfg/config.c
+++ b/libs/kfg/config.c
@@ -1597,9 +1597,14 @@ static rc_t _printNodeData(const char *name, const char *data, uint32_t dlen,
}
}
-static rc_t KConfigNodePrint(const KConfigNode *self, int indent,
+
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
+
+static rc_t KConfigNodePrintWithIncluded (const KConfigNode *self, int indent,
const char* root, bool debug, bool native, const char* aFullpath,
- PrintBuff *pb, uint32_t skipCount, va_list args)
+ PrintBuff *pb, uint32_t skipCount, va_list args,
+ const KConfig * withIncluded )
{
rc_t rc = 0;
KNamelist* names = NULL;
@@ -1616,9 +1621,8 @@ static rc_t KConfigNodePrint(const KConfigNode *self, int indent,
bool found = false;
uint32_t i = 0;
va_list args_copy;
- if (skipCount > 0) {
+ if (skipCount > 0)
va_copy(args_copy, args);
- }
for (i = 0; i < skipCount; ++i) {
const char *skip = va_arg(args_copy, const char*);
if (string_cmp(skip, string_measure(skip, NULL), root,
@@ -1631,13 +1635,36 @@ static rc_t KConfigNodePrint(const KConfigNode *self, int indent,
break;
}
}
- if (skipCount > 0) {
+ if (skipCount > 0)
va_end(args_copy);
- }
- if (found) {
+ if (found)
return rc;
- }
rc = PrintBuffPrint(pb, "<%s>", root);
+ if ( withIncluded ) {
+// bool hasAny = false;
+ uint32_t count = 0;
+ KNamelist * names = NULL;
+ rc_t rc = KConfigListIncluded ( withIncluded, & names );
+ if ( rc == 0 )
+ rc = KNamelistCount ( names, & count );
+ if ( rc == 0 ) {
+ uint32_t i = 0;
+ rc = printIndent(indent, pb);
+ PrintBuffPrint ( pb, "\n <ConfigurationFiles>\n" );
+ for ( i = 0; i < count && rc == 0; ++i ) {
+ const char * name = NULL;
+ if ( rc == 0 )
+ rc = KNamelistGet(names, i, &name);
+ if (rc == 0) {
+ PrintBuffPrint ( pb, "%s\n", name );
+// hasAny = true;
+ }
+ }
+ rc = printIndent(indent, pb);
+ PrintBuffPrint ( pb, " </ConfigurationFiles>" );
+ }
+ RELEASE ( KNamelist, names );
+ }
}
}
@@ -1664,9 +1691,8 @@ static rc_t KConfigNodePrint(const KConfigNode *self, int indent,
}
if (rc == 0) {
- if (count > 0 && !native) {
+ if (count > 0 && !native)
rc = PrintBuffPrint(pb, "\n");
- }
for (i = 0; i < count; ++i) {
char *fullpath = NULL;
const char* name = NULL;
@@ -1693,8 +1719,8 @@ static rc_t KConfigNodePrint(const KConfigNode *self, int indent,
}
if (rc == 0) {
if (! isdigit(name[0])) {
- KConfigNodePrint(node, indent + 1, name,
- debug, native, fullpath, pb, skipCount, args);
+ KConfigNodePrintWithIncluded(node, indent + 1, name,
+ debug, native, fullpath, pb, skipCount, args, NULL );
}
else {
/* XML node names cannot start with a number */
@@ -1706,8 +1732,9 @@ static rc_t KConfigNodePrint(const KConfigNode *self, int indent,
}
else {
string_printf(dname, dsize, NULL, "_%s", name);
- KConfigNodePrint(node, indent + 1, dname,
- debug, native, fullpath, pb, skipCount, args);
+ KConfigNodePrintWithIncluded ( node, indent + 1, dname,
+ debug, native, fullpath, pb, skipCount, args,
+ NULL );
free(dname);
}
}
@@ -1730,14 +1757,24 @@ static rc_t KConfigNodePrint(const KConfigNode *self, int indent,
return rc;
}
+static rc_t KConfigNodePrint(const KConfigNode *self, int indent,
+ const char* root, bool debug, bool native, const char* aFullpath,
+ PrintBuff *pb, uint32_t skipCount, va_list args, const KConfig* cfg )
+{
+ return KConfigNodePrintWithIncluded ( self, indent, root, debug, native,
+ aFullpath, pb, skipCount, args, cfg );
+}
+
static rc_t CC KConfigPrintImpl(const KConfig* self,
int indent, const char *root, bool debug, bool native,
PrintBuff *pb, uint32_t skipCount, va_list args)
{
+ const KConfig * withIncluded = NULL;
rc_t rc = 0;
if (root == NULL) {
root = "Config";
+ withIncluded = self;
}
if (self == NULL) {
@@ -1751,19 +1788,15 @@ static rc_t CC KConfigPrintImpl(const KConfig* self,
rc = KConfigOpenNodeRead(self, &node, "/");
DISP_RC2(rc, "KConfigOpenNodeRead()", "/");
}
- if (rc == 0) {
- KConfigNodePrint
- (node, indent, root, debug, native, "", pb, skipCount, args);
- }
+ if (rc == 0)
+ KConfigNodePrint ( node, indent, root, debug, native, "", pb,
+ skipCount, args, withIncluded );
KConfigNodeRelease(node);
}
return rc;
}
-#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
- if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
-
LIB_EXPORT rc_t CC KConfigPrintDebug(const KConfig* self, const char *path) {
rc_t rc = 0;
@@ -2191,7 +2224,7 @@ rc_t path_to_magic_file ( const KConfig *self, char *path, size_t buffer_size, s
LIB_EXPORT rc_t CC KConfigCommit ( KConfig *self )
{
rc_t rc;
- size_t path_size;
+ size_t path_size = 0;
char magic_file_path [ 4096 ];
if ( self == NULL )
@@ -2986,7 +3019,7 @@ static rc_t _KConfigFixRepeatedDrives(KConfig *self,
const KDirectory *pdir, bool *updated)
{
rc_t rc = 0;
- const KDirectory *dir = pdir;
+ KDirectory * dir = ( KDirectory * ) pdir;
KConfigNode *user = NULL;
if (dir == NULL) {
rc = KDirectoryNativeDir(&dir);
@@ -4196,7 +4229,13 @@ static rc_t _KConfigDBGapRepositoryNodes(KConfig *self,
rc = _KConfigNodeUpdateChild(rep, "apps/file/volumes/flat", "files");
}
if (rc == 0) {
- rc = _KConfigNodeUpdateChild(rep, "apps/sra/volumes/sraFlat", "sra");
+ const char name [] = "apps/sra/volumes/sraFlat";
+ const KConfigNode * node = NULL;
+ rc = KConfigNodeOpenNodeRead ( rep, & node, name );
+ if ( rc != 0 )
+ rc = _KConfigNodeUpdateChild ( rep, name, "sra" );
+ else
+ KConfigNodeRelease ( node );
}
if (rc == 0) {
diff --git a/libs/kfg/kart.c b/libs/kfg/kart.c
index 22b14c3..484a15d 100644
--- a/libs/kfg/kart.c
+++ b/libs/kfg/kart.c
@@ -24,7 +24,7 @@
*
*/
-#include <kfg/kart.h>
+#include <kfg/kart-priv.h> /* KartMake2 */
#include <kfs/directory.h> /* KDirectoryOpenFileRead */
#include <kfs/file.h> /* KFile */
@@ -35,6 +35,7 @@
#include <klib/rc.h>
#include <klib/refcount.h> /* KRefcount */
#include <klib/out.h> /* OUTMSG */
+#include <klib/vector.h> /* Vector */
#include <strtol.h> /* strtou64 */
#include <sysalloc.h>
@@ -46,29 +47,126 @@
#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false)
+typedef enum {
+ eVersion1, // from kart 0x01000000
+ eVersion2, // from kart 0x02000000
+} Eversion;
+
struct KartItem {
KRefcount refcount;
const Kart *dad;
-/* String typeId; */
+ Eversion version;
+
String projId;
String itemId;
- String accession;
+ String accession; /* 1.0 */
String name;
String itemDesc;
+
+ /* 2.0 */
+ String objType;
+ String path;
+ String size;
};
static void KartItemWhack(KartItem *self) {
assert(self);
- KartRelease(self->dad);
+ if ( self -> version < eVersion2 ) {
+ KartRelease(self->dad);
+ } else {
+ free ( ( void * ) self -> projId . addr );
+ }
memset(self, 0, sizeof *self);
free(self);
}
+static rc_t KartItemMake2
+ ( KartItem ** self, const char * buffer, size_t size )
+{
+ bool BUG = false;
+ rc_t rc = 0;
+ KartItem * obj = NULL;
+ int i = 0;
+ assert ( self );
+ obj = calloc ( 1, sizeof * obj );
+ if ( obj == NULL ) {
+ return RC ( rcKFG, rcData, rcAllocating, rcMemory, rcExhausted );
+ }
+ obj -> version = eVersion2; /* 0x02000000; */
+ for ( i = 0; ; ++i ) {
+ size_t l = 0;
+ String * next = NULL;
+ const char * p = string_chr ( buffer, size, '|' );
+ if ( p == NULL ) {
+ if ( i != 7 ) {
+ rc = RC(rcKFG, rcFile, rcParsing, rcFile, rcInsufficient);
+ break;
+ }
+ l = size;
+ }
+ else {
+ l = p - buffer;
+ }
+ switch ( i ) {
+ case 0:
+ next = & obj -> projId;
+ break;
+ case 1:
+ next = & obj -> objType;
+ break;
+ case 2:
+ next = & obj -> itemId;
+ break;
+ case 3:
+ next = & obj -> name;
+ break;
+ case 4:
+ next = & obj -> path;
+ break;
+ case 5:
+ next = & obj -> size;
+ break;
+ case 6:
+ next = & obj -> itemDesc;
+ break;
+ case 7:
+ BUG = true;
+ break;
+ default:
+ rc = RC ( rcKFG, rcFile, rcParsing, rcFile, rcExcessive );
+ break;
+ }
+ if ( ! BUG ) {
+ assert ( next );
+ StringInit ( next, buffer, l, ( uint32_t ) l );
+ if ( l > size ) {
+ rc = RC ( rcKFG, rcFile, rcParsing, rcFile, rcInvalid );
+ }
+ }
+ if ( size == l ) {
+ break;
+ }
+ ++ l;
+ buffer += l;
+ size -= l;
+ }
+ if (rc == 0) {
+ KRefcountInit ( & obj -> refcount, 1,
+ "KartItem", "KartItemMake2", "kartitem" );
+ * self = obj;
+ }
+ else {
+ free ( obj );
+ obj = NULL;
+ }
+ return rc;
+}
+
/* AddRef
* Release
* all objects are reference counted
@@ -137,14 +235,18 @@ static rc_t StringAsUint64(const String *self, uint64_t *pid) {
return 0;
}
-LIB_EXPORT rc_t CC KartItemProjIdNumber(const KartItem *self, uint64_t *pid) {
+LIB_EXPORT
+rc_t CC KartItemProjIdNumber(const KartItem *self, uint64_t *pid)
+{
if (self == NULL) {
return RC(rcKFG, rcFile, rcAccessing, rcSelf, rcNull);
}
return StringAsUint64(&self->projId, pid);
}
-LIB_EXPORT rc_t CC KartItemItemIdNumber(const KartItem *self, uint64_t *pid) {
+LIB_EXPORT
+rc_t CC KartItemItemIdNumber(const KartItem *self, uint64_t *pid)
+{
if (self == NULL) {
return RC(rcKFG, rcFile, rcAccessing, rcSelf, rcNull);
}
@@ -165,7 +267,8 @@ static rc_t KartItemCheck(const KartItem *self, const String **elem) {
return 0;
}
-LIB_EXPORT rc_t CC KartItemProjId(const KartItem *self, const String **elem)
+LIB_EXPORT
+rc_t CC KartItemProjId(const KartItem *self, const String **elem)
{
rc_t rc = KartItemCheck(self, elem);
if (rc == 0) {
@@ -173,7 +276,8 @@ LIB_EXPORT rc_t CC KartItemProjId(const KartItem *self, const String **elem)
}
return rc;
}
-LIB_EXPORT rc_t CC KartItemItemId(const KartItem *self, const String **elem)
+LIB_EXPORT
+rc_t CC KartItemItemId(const KartItem *self, const String **elem)
{
rc_t rc = KartItemCheck(self, elem);
if (rc == 0) {
@@ -181,7 +285,8 @@ LIB_EXPORT rc_t CC KartItemItemId(const KartItem *self, const String **elem)
}
return rc;
}
-LIB_EXPORT rc_t CC KartItemAccession(const KartItem *self, const String **elem)
+LIB_EXPORT
+rc_t CC KartItemAccession(const KartItem *self, const String **elem)
{
rc_t rc = KartItemCheck(self, elem);
if (rc == 0) {
@@ -189,7 +294,8 @@ LIB_EXPORT rc_t CC KartItemAccession(const KartItem *self, const String **elem)
}
return rc;
}
-LIB_EXPORT rc_t CC KartItemName(const KartItem *self, const String **elem)
+LIB_EXPORT
+rc_t CC KartItemName(const KartItem *self, const String **elem)
{
rc_t rc = KartItemCheck(self, elem);
if (rc == 0) {
@@ -197,7 +303,8 @@ LIB_EXPORT rc_t CC KartItemName(const KartItem *self, const String **elem)
}
return rc;
}
-LIB_EXPORT rc_t CC KartItemItemDesc(const KartItem *self, const String **elem)
+LIB_EXPORT
+rc_t CC KartItemItemDesc(const KartItem *self, const String **elem)
{
rc_t rc = KartItemCheck(self, elem);
if (rc == 0) {
@@ -205,6 +312,30 @@ LIB_EXPORT rc_t CC KartItemItemDesc(const KartItem *self, const String **elem)
}
return rc;
}
+LIB_EXPORT rc_t CC KartItemObjType (const KartItem *self, const String **elem )
+{
+ rc_t rc = KartItemCheck(self, elem);
+ if (rc == 0) {
+ *elem = &self->objType;
+ }
+ return rc;
+}
+LIB_EXPORT rc_t CC KartItemPath (const KartItem *self, const String **elem )
+{
+ rc_t rc = KartItemCheck(self, elem);
+ if (rc == 0) {
+ *elem = &self->path;
+ }
+ return rc;
+}
+LIB_EXPORT rc_t CC KartItemSize (const KartItem *self, const String **elem )
+{
+ rc_t rc = KartItemCheck(self, elem);
+ if (rc == 0) {
+ *elem = &self->size;
+ }
+ return rc;
+}
/*LIB_EXPORT rc_t CC KartItemTypeId(const KartItem *self, const String **elem)
{
rc_t rc = KartItemCheck(self, elem);
@@ -226,18 +357,30 @@ LIB_EXPORT rc_t CC KartItemPrint(const KartItem *self) { /* AA-833 */
struct Kart {
KRefcount refcount;
- KDataBuffer mem;
+ Eversion version;
+ /* version eVersion1 0x01000000 */
+ KDataBuffer mem;
const char *text;
uint64_t len;
-
uint16_t itemsProcessed;
+
+ /* version eVersion2 0x02000000 */
+ Vector rows;
};
+static void whackKartItem ( void * self, void * ignore ) {
+ KartItemRelease ( ( KartItem * ) self);
+}
+
static void KartWhack(Kart *self) {
assert(self);
- KDataBufferWhack(&self->mem);
+ if ( self -> version < eVersion2 ) {
+ KDataBufferWhack(&self->mem);
+ } else {
+ VectorWhack ( & self -> rows, whackKartItem, NULL );
+ }
memset(self, 0, sizeof *self);
@@ -364,14 +507,25 @@ static rc_t KartItemInitFromKartRow(const Kart *self, const KartItem **item,
}
LIB_EXPORT rc_t CC KartPrint(const Kart *self) {
- uint32_t len = 0;
-
- if (self == NULL) {
+ if (self == NULL)
return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
- }
- len = (uint32_t)self->mem.elem_count;;
- return OUTMSG(("%.*s", len, self->mem.base));
+ if ( self -> version == eVersion1 ) {
+ uint32_t l = ( uint32_t ) self -> mem . elem_count;
+ OUTMSG ( ( "%.*s", l, self -> mem.base ) );
+ }
+ else {
+ uint32_t i = 0;
+ uint32_t l = VectorLength ( & self -> rows );
+ for ( i = 0; i < l; ++ i ) {
+ KartItem * result = VectorGet ( & self -> rows, i );
+ assert ( result );
+ OUTMSG ( ( "%S|%S|%S|%S|%S|%S|%S\n", & result -> projId,
+ & result -> objType, & result -> itemId, & result -> name,
+ & result -> path, & result -> size, & result -> itemDesc ) );
+ }
+ }
+ return 0;
}
LIB_EXPORT rc_t CC KartPrintNumbered(const Kart *self) {
@@ -388,6 +542,9 @@ LIB_EXPORT rc_t CC KartPrintNumbered(const Kart *self) {
return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
}
+ if ( self -> version > eVersion1 )
+ return RC ( rcKFG, rcFile, rcAccessing, rcInterface, rcBadVersion );
+
remaining = (uint32_t)self->mem.elem_count;
start = self->mem.base;
@@ -454,7 +611,10 @@ LIB_EXPORT rc_t CC KartPrintNumbered(const Kart *self) {
return rc;
}
-LIB_EXPORT rc_t CC KartMakeNextItem(Kart *self, const KartItem **item) {
+LIB_EXPORT
+rc_t CC KartMakeNextItem ( const Kart * cself, const KartItem **item )
+{
+ Kart * self = ( Kart * ) cself;
size_t len = 0;
const char *line = NULL;
const char *next = NULL;
@@ -467,45 +627,63 @@ LIB_EXPORT rc_t CC KartMakeNextItem(Kart *self, const KartItem **item) {
return RC(rcKFG, rcFile, rcLoading, rcSelf, rcNull);
}
- while (self->len > 0
- && (self->text[0] == '\r' || self->text[0] == '\n'))
- {
- ++self->text;
- --self->len;
- }
+ if ( self -> version < eVersion2 ) {
+ while (self->len > 0
+ && (self->text[0] == '\r' || self->text[0] == '\n'))
+ {
+ ++self->text;
+ --self->len;
+ }
- line = self->text;
- next = string_chr(self->text, self->len, '\n');
- if (next == NULL) {
- return RC(rcKFG, rcFile, rcLoading, rcFile, rcInsufficient);
- }
+ line = self->text;
+ next = string_chr(self->text, self->len, '\n');
+ if (next == NULL) {
+ return RC(rcKFG, rcFile, rcLoading, rcFile, rcInsufficient);
+ }
- len = next - self->text;
- if (*(next - 1) == '\r') {
- --len;
- }
+ len = next - self->text;
+ if (*(next - 1) == '\r') {
+ --len;
+ }
- if (self->len >= (uint64_t) (next - self->text + 1) ){
- self->len -= next - self->text + 1;
- }
- else {
- OUTMSG(("WARNING: STRING OVERFLOW DURING KART ROW PARSING"));
- self->len = 0;
- }
+ if (self->len >= (uint64_t) (next - self->text + 1) ){
+ self->len -= next - self->text + 1;
+ }
+ else {
+ OUTMSG(("WARNING: STRING OVERFLOW DURING KART ROW PARSING"));
+ self->len = 0;
+ }
- self->text = next + 1;
+ self->text = next + 1;
- {
- const char end[] = "$end";
- if (string_cmp(line, len, end, sizeof end - 1, sizeof end - 1) == 0) {
- return 0;
+ {
+ const char end[] = "$end";
+ if (string_cmp(line, len, end, sizeof end - 1, sizeof end - 1) == 0)
+ {
+ return 0;
+ }
}
- }
- return KartItemInitFromKartRow(self, item, line, len);
+ return KartItemInitFromKartRow(self, item, line, len);
+ } else {
+ rc_t rc = 0;
+ uint32_t l = VectorLength ( & self -> rows );
+ if ( self -> len < l ) {
+ KartItem * result = VectorGet ( & self -> rows, self -> len ++ );
+ if ( result != NULL ) {
+ rc = KartItemAddRef ( result );
+ if ( rc == 0 ) {
+ * item = result;
+ }
+ }
+ }
+ return rc;
+ }
}
-static rc_t decode_kart(KDataBuffer *mem, const KFile *orig, size_t hdr_sz) {
+static
+rc_t decode_kart(KDataBuffer *mem, const KFile *orig, size_t hdr_sz)
+{
rc_t rc = 0;
size_t num_read;
uint64_t eof;
@@ -669,7 +847,56 @@ KFG_EXTERN rc_t CC KartMakeText(const struct KDirectory *dir, const char *path,
}
#endif
-LIB_EXPORT rc_t KartMake(const KDirectory *dir, const char *path,
+LIB_EXPORT rc_t CC KartMake2 ( Kart ** kart ) {
+ Kart * obj = NULL;
+
+ if ( kart == NULL )
+ return RC ( rcKFG, rcFile, rcReading, rcParam, rcNull );
+
+ obj = calloc ( 1, sizeof * obj );
+ if ( obj == NULL )
+ return RC ( rcKFG, rcData, rcAllocating, rcMemory, rcExhausted );
+
+ obj -> version = eVersion2;
+
+ KRefcountInit ( & obj->refcount, 1, "Kart", "KartMake2", "kart" );
+
+ * kart = obj;
+
+ return 0;
+}
+
+LIB_EXPORT rc_t CC KartAddRow ( Kart * self, const char * row, size_t size ) {
+ if ( self == NULL )
+ return RC ( rcKFG, rcFile, rcUpdating, rcSelf, rcNull );
+ if ( row == NULL )
+ return RC ( rcKFG, rcFile, rcUpdating, rcParam, rcNull );
+
+ if ( self -> version < eVersion2 )
+ return RC ( rcKFG, rcFile, rcUpdating, rcInterface, rcBadVersion );
+
+ {
+ rc_t rc = 0;
+
+ KartItem * item = NULL;
+
+ const char * p = string_dup ( row, size );
+ if ( p == NULL )
+ return RC ( rcKFG, rcFile, rcUpdating, rcMemory, rcExhausted );
+
+ rc = KartItemMake2 ( & item, p, size );
+ if ( rc == 0 ) {
+ rc = VectorAppend ( & self -> rows, NULL, item );
+
+ if ( rc != 0 )
+ KartItemRelease ( item );
+ }
+
+ return rc;
+ }
+}
+
+LIB_EXPORT rc_t CC KartMake(const KDirectory *dir, const char *path,
Kart **kart, bool *isKart)
{
rc_t rc = 0;
diff --git a/libs/kfg/keystore.c b/libs/kfg/keystore.c
index 7e3b98a..7e17b3d 100644
--- a/libs/kfg/keystore.c
+++ b/libs/kfg/keystore.c
@@ -272,6 +272,12 @@ static rc_t CC KKeyStoreGetKeyInt(const KKeyStore* self, const char* obj_key,
rc = KRepositoryEncryptionKeyFile ( protected, path, sizeof(path), NULL );
if ( rc == 0 && path [ 0 ] != 0 )
rc = KEncryptionKeyMakeFromFile(path, enc_key);
+ else {
+ rc = KRepositoryEncryptionKey ( protected,
+ path, sizeof(path), NULL );
+ if ( rc == 0 )
+ rc = KEncryptionKeyMake ( path, enc_key );
+ }
rc2 = KRepositoryRelease ( protected );
if (rc == 0)
diff --git a/libs/kfs/Makefile b/libs/kfs/Makefile
index eaed481..ac5343c 100644
--- a/libs/kfs/Makefile
+++ b/libs/kfs/Makefile
@@ -120,7 +120,8 @@ KFS_CMN = \
lockfile \
syslockfile \
cacheteefile \
- from_to_namelist
+ from_to_namelist \
+ limitfile
KFS_SRC = \
mmap \
diff --git a/libs/kfs/cacheteefile.c b/libs/kfs/cacheteefile.c
index b4a7c0d..002462b 100644
--- a/libs/kfs/cacheteefile.c
+++ b/libs/kfs/cacheteefile.c
@@ -87,7 +87,6 @@ typedef struct CacheStatistic
uint64_t requests_below_32k;
uint64_t requests_consecutive;
uint64_t requests_in_first32k;
-
uint64_t prev_pos;
uint64_t requests_same_pos;
uint64_t requests_same_pos_and_len;
@@ -190,6 +189,7 @@ typedef struct KCacheTeeFile
#else
uint8_t volatile * bitmap; /* the bitmap of cached blocks */
#endif
+
uint64_t bitmap_bytes; /* how many bytes do we need to store the bitmap */
#if USE_BUFFER_POOL
@@ -841,6 +841,14 @@ static void set_bitmap( const KCacheTeeFile *cself, uint64_t start_block, uint64
}
+static rc_t switch_to_read_only( const KCacheTeeFile *cself, rc_t rc )
+{
+ KCacheTeeFile *self = ( KCacheTeeFile * )cself;
+ self->local_read_only = true;
+ LOGERR( klogInt, rc, "switching cache-tee-file to read-only" );
+ return 0;
+}
+
static rc_t write_bitmap( const KCacheTeeFile *cself, uint64_t block )
{
rc_t rc;
@@ -866,8 +874,13 @@ static rc_t write_bitmap( const KCacheTeeFile *cself, uint64_t block )
#endif
if ( rc != 0 )
{
+ /* it can happen that we are not able to write to the bitmap because we run out of space
+ on the local filesystem. */
+ rc = switch_to_read_only( cself, rc );
+ /*
PLOGERR( klogErr, ( klogErr, rc, "cannot write local-file-bitmap block $(block) at $(pos) $(to_write) bytes",
"block=%lu,pos=%lu,to_write=%zu", block, pos, to_write ) );
+ */
}
return rc;
}
@@ -928,7 +941,7 @@ static rc_t rd_remote_wr_local( const KCacheTeeFile *cself, uint64_t pos,
size_t bytes_read;
*num_read = 0;
rc = KFileReadAll( cself->remote, pos, buffer, bsize, &bytes_read );
- if ( rc != 0 || bytes_read == 0) /** try again **/
+ if ( rc != 0 || bytes_read == 0 ) /** try again **/
{
rc = KFileReadAll( cself->remote, pos, buffer, bsize, &bytes_read );
if ( rc == 0 && bytes_read == 0 )
@@ -941,7 +954,13 @@ static rc_t rd_remote_wr_local( const KCacheTeeFile *cself, uint64_t pos,
if ( cself->local_read_only )
*num_read = bytes_read;
else
+ {
+ /* it can happen that we are running out of space in the local filesystem,
+ that means we cannot write ( any more ) */
rc = KFileWriteAll( cself->local, pos, buffer, bytes_read, num_read );
+ if ( rc != 0 )
+ rc = switch_to_read_only( cself, rc );
+ }
}
}
return rc;
@@ -1013,55 +1032,70 @@ static rc_t KCacheTeeFileRead_simple2( const KCacheTeeFile *cself, uint64_t pos,
else if ( IS_CACHE_BIT( cself, block ) )
{
uint64_t fpos = block * cself->block_size;
- int64_t fbsize = cself -> remote_size - fpos;
- size_t nread = 0;
+ if ( fpos < cself -> remote_size )
+ {
+ int64_t fbsize = cself -> remote_size - fpos;
+ size_t nread = 0;
- if( fbsize > cself->block_size )
- fbsize = cself -> block_size;
+ if( fbsize > cself->block_size )
+ fbsize = cself -> block_size;
- rc = KFileReadAll( cself->local, fpos, scratch_buffer, fbsize, &nread );
- if ( rc == 0 )
- {
- int i;
- uint64_t *b = ( uint64_t* )scratch_buffer;
- first_block_in_scratch = block;
- valid_scratch_bytes = nread;
-
- if ( block != salvage_block )
- { /** check for fully space page, but don't do it in infinite loop **/
- for ( i = 0; i < ( nread/ sizeof( *b ) ) && b [ i]==0; i++ ) { }
- if ( i == ( nread / sizeof( *b ) ) )
- {
- rc = rd_remote_wr_local( cself, block*cself->block_size, scratch_buffer, fbsize, &nread );
- if ( rc == 0 )
- salvage_block = block;
- }
- else
- {
- salvage_block = -1;
+ rc = KFileReadAll( cself->local, fpos, scratch_buffer, fbsize, &nread );
+ if ( rc == 0 )
+ {
+ int i;
+ uint64_t *b = ( uint64_t* )scratch_buffer;
+ first_block_in_scratch = block;
+ valid_scratch_bytes = nread;
+
+ if ( block != salvage_block )
+ { /** check for fully space page, but don't do it in infinite loop **/
+ for ( i = 0; i < ( nread/ sizeof( *b ) ) && b [ i]==0; i++ ) { }
+ if ( i == ( nread / sizeof( *b ) ) )
+ {
+ rc = rd_remote_wr_local( cself, block*cself->block_size, scratch_buffer, fbsize, &nread );
+ if ( rc == 0 )
+ salvage_block = block;
+ }
+ else
+ {
+ salvage_block = -1;
+ }
}
}
}
+ else
+ {
+ to_read_total = 0;
+ }
}
else
{
uint64_t fpos = block * cself->block_size;
- int64_t fbsize = cself->remote_size - fpos;
- size_t nread = 0;
-
- if ( fbsize > cself->block_size )
- fbsize = cself->block_size;
- rc = rd_remote_wr_local( cself, fpos, scratch_buffer, fbsize, &nread );
- if ( rc == 0 )
+ if ( fpos < cself -> remote_size )
{
- first_block_in_scratch = block;
- valid_scratch_bytes = nread;
- if ( !cself->local_read_only )
+ int64_t fbsize = cself->remote_size - fpos;
+ size_t nread = 0;
+
+ if ( fbsize > cself->block_size )
+ fbsize = cself->block_size;
+
+ rc = rd_remote_wr_local( cself, fpos, scratch_buffer, fbsize, &nread );
+ if ( rc == 0 )
{
- set_bitmap( cself, block, 1 );
- rc = write_bitmap( cself, block );
+ first_block_in_scratch = block;
+ valid_scratch_bytes = nread;
+ if ( !cself->local_read_only )
+ {
+ set_bitmap( cself, block, 1 );
+ rc = write_bitmap( cself, block );
+ }
}
}
+ else
+ {
+ to_read_total = 0;
+ }
}
}
@@ -1311,6 +1345,13 @@ static KFile_vt_v1 vtKCacheTeeFile =
/* end minor version 0 methods */
};
+static rc_t hand_out_remote_file_as_tee_file( struct KFile const **tee, struct KFile const *remote )
+{
+ rc_t rc = KFileAddRef( remote );
+ if ( rc == 0 )
+ *tee = remote;
+ return rc;
+}
static rc_t make_cache_tee( struct KDirectory *self, struct KFile const **tee,
struct KFile const *remote, struct KFile *local, uint32_t blocksize, bool read_only, const char *path )
@@ -1447,11 +1488,13 @@ static rc_t make_cache_tee( struct KDirectory *self, struct KFile const **tee,
}
}
free ( cf );
+ /* if we arrived here, we do not have enough space on the local filesystem */
+ rc = hand_out_remote_file_as_tee_file( tee, remote );
+ LOGERR( klogInt, rc, "skipping the cache-tee completely" );
}
return rc;
}
-
static rc_t make_read_only_cache_tee( struct KDirectory *self,
struct KFile const **tee, struct KFile const *remote, uint32_t blocksize, const char *path )
{
@@ -1459,14 +1502,6 @@ static rc_t make_read_only_cache_tee( struct KDirectory *self,
rc_t rc = KDirectoryOpenFileRead( self, &local, "%s.cache", path );
if ( rc == 0 )
rc = make_cache_tee( self, tee, remote, ( struct KFile * )local, blocksize, true, path );
- else
- {
- /* we cannot even open the local cache in read-only mode,
- we give up - and return a reference to the remote file... */
- rc = KFileAddRef( remote );
- if ( rc == 0 )
- *tee = remote;
- }
return rc;
}
diff --git a/libs/kfs/from_to_namelist.c b/libs/kfs/from_to_namelist.c
index b96028d..285ca2c 100644
--- a/libs/kfs/from_to_namelist.c
+++ b/libs/kfs/from_to_namelist.c
@@ -428,7 +428,7 @@ LIB_EXPORT rc_t CC WriteNameListToKFile( struct KFile * self, const VNamelist *
LIB_EXPORT rc_t CC WriteNamelistToFileByName( const VNamelist * namelist, const char * filename,
- const char * delim )
+ const char * delim )
{
rc_t rc;
if ( namelist == NULL || filename == NULL || delim == NULL )
@@ -452,3 +452,56 @@ LIB_EXPORT rc_t CC WriteNamelistToFileByName( const VNamelist * namelist, const
}
return rc;
}
+
+
+typedef struct dir_entry_ctx
+{
+ VNamelist * namelist;
+ bool add_files;
+ bool add_dirs;
+} dir_entry_ctx;
+
+
+static rc_t CC on_dir_entry( const KDirectory * dir, uint32_t type, const char * name, void * data )
+{
+ rc_t rc = 0;
+ if ( name != NULL && data != NULL && name[ 0 ] != 0 && name[ 0 ] != '.' )
+ {
+ dir_entry_ctx * dec = data;
+ bool is_dir = ( ( type & ~kptAlias ) == kptDir );
+ bool is_file = ( ( type & ~kptAlias ) == kptFile );
+ if ( ( dec->add_dirs && is_dir ) || ( dec->add_files && is_file ) )
+ rc = VNamelistAppend( dec->namelist, name );
+ }
+ return rc;
+}
+
+LIB_EXPORT rc_t CC ReadDirEntriesIntoToNamelist( VNamelist ** namelist, const KDirectory * dir,
+ bool perform_sort, bool add_files, bool add_dirs, const char * path )
+{
+ rc_t rc;
+ if ( namelist == NULL || dir == NULL )
+ rc = RC( rcFS, rcFile, rcValidating, rcParam, rcNull );
+ else
+ {
+ dir_entry_ctx dec;
+
+ *namelist = NULL;
+ rc = VNamelistMake( &dec.namelist, 25 );
+ if ( rc == 0 )
+ {
+ dec.add_files = add_files;
+ dec.add_dirs = add_dirs;
+
+ rc = KDirectoryVisit( dir, false, on_dir_entry, &dec, "%s", path );
+ if ( rc == 0 && perform_sort )
+ VNamelistReorder( dec.namelist, false );
+
+ if ( rc == 0 )
+ *namelist = dec.namelist;
+ else
+ VNamelistRelease( dec.namelist );
+ }
+ }
+ return rc;
+}
diff --git a/libs/kfs/gzip.c b/libs/kfs/gzip.c
index beb22fb..048892e 100644
--- a/libs/kfs/gzip.c
+++ b/libs/kfs/gzip.c
@@ -31,6 +31,7 @@ struct KGZipFile;
#include <kfs/impl.h> /* KFile_vt_v1 */
#include <kfs/gzip.h> /* KFileMakeGzipFor... */
#include <klib/debug.h>
+#include <klib/status.h>
#include <klib/rc.h>
#include <klib/out.h>
#include <sysalloc.h>
@@ -46,6 +47,15 @@ struct KGZipFile;
#define GZIP_DEBUG(msg)
#endif
+#if _DEBUGGING && 0
+/* set limit to 1MB */
+#define USE_FILE_LIMIT 0x100000
+#endif
+
+#ifdef USE_FILE_LIMIT
+#include <kfs/limitfile.h>
+#endif
+
/***************************************************************************************/
/* Gzip File */
/***************************************************************************************/
@@ -416,52 +426,66 @@ LIB_EXPORT rc_t CC KFileMakeGzipForWrite( struct KFile **result,
struct KFile *file )
{
rc_t rc;
- z_stream* strm;
- KGZipFile *obj;
-
- if ( result == NULL || file == NULL )
- return RC ( rcFS, rcFile, rcConstructing, rcParam, rcNull );
-
- obj = (KGZipFile*) malloc(sizeof(KGZipFile));
- if (!obj)
- return RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted );
-
- rc = KFileInit(&obj->dad, (const KFile_vt*) &s_vtKFile_OutGz, "KGZipFile", "no-name", false, true);
- if (rc != 0) {
- free(obj);
- return rc;
- }
- strm = &obj->strm;
- strm->zalloc = Z_NULL;
- strm->zfree = Z_NULL;
- strm->opaque = Z_NULL;
- strm->avail_in = 0;
- strm->next_in = Z_NULL;
-
- /* TBD - this should check gzlib error codes */
- if (deflateInit2(strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, WINDOW_BITS,
- 8, /* The default value for the memLevel parameter is 8 */
- Z_DEFAULT_STRATEGY) != Z_OK)
+ if ( result == NULL )
+ rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcNull );
+ else
{
- free(obj);
- return RC ( rcFS, rcFile, rcConstructing, rcNoObj, rcUnknown );
- }
+ if ( file == NULL )
+ rc = RC ( rcFS, rcFile, rcConstructing, rcParam, rcNull );
+ else if ( ! file -> write_enabled )
+ {
+ if ( file -> read_enabled )
+ rc = RC ( rcFS, rcFile, rcConstructing, rcFile, rcReadonly );
+ else
+ rc = RC ( rcFS, rcFile, rcConstructing, rcFile, rcNoPerm );
+ }
+ else
+ {
+ KGZipFile * obj = calloc ( 1, sizeof * obj );
+ if ( obj == NULL )
+ rc = RC ( rcFS, rcFile, rcConstructing, rcMemory, rcExhausted );
+ else
+ {
+ rc = KFileInit( & obj->dad, (const KFile_vt*) &s_vtKFile_OutGz,
+ "KGZipFile", "no-name", false, true );
+ if ( rc == 0 )
+ {
+ /* The default value for the memLevel parameter is 8 */
+ if ( deflateInit2 (&obj->strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
+ WINDOW_BITS, 8, Z_DEFAULT_STRATEGY) != Z_OK )
+ {
+ rc = RC ( rcFS, rcFile, rcConstructing, rcNoObj, rcUnknown );
+ }
+ else
+ {
+#if USE_FILE_LIMIT
+ KFile * limitFile;
+ STATUS ( STAT_USR, "wrapping gzip output file in a limit file with limit %u bytes per block.\n", USE_FILE_LIMIT );
+ rc = KFileMakeLimitFile ( & limitFile, file, USE_FILE_LIMIT );
+ if ( rc == 0 )
+ {
+ obj -> file = limitFile;
+ * result = & obj -> dad;
+ return 0;
+ }
+#else
+ rc = KFileAddRef ( file );
+ if ( rc == 0 )
+ {
+ obj -> file = file;
+ * result = & obj -> dad;
+ return 0;
+ }
+#endif
+ }
+ }
- obj->myPosition = 0;
- obj->filePosition = 0;
- obj->completed = false;
+ free ( obj );
+ }
+ }
- rc = KFileAddRef(file);
- if ( rc != 0 )
- {
- obj->file = NULL;
- KGZipFile_OutDestroy ( obj );
- }
- else
- {
- obj->file = file;
- *result = &obj->dad;
+ * result = NULL;
}
return rc;
@@ -568,7 +592,7 @@ static int s_GzipAndWrite ( KGZipFile *self,
assert( ret != Z_STREAM_ERROR ); /* state not clobbered */
have = sizeof( self->buff ) - strm->avail_out;
written = 0;
- *rc = KFileWrite( self->file, self->filePosition, self->buff, have, &written );
+ *rc = KFileWriteAll( self->file, self->filePosition, self->buff, have, &written );
/* this is wrong - Z_ERRNO would tell us to check errno for error
but the error is in *rc */
if ( *rc != 0 )
diff --git a/libs/kfs/limitfile.c b/libs/kfs/limitfile.c
new file mode 100644
index 0000000..174bc11
--- /dev/null
+++ b/libs/kfs/limitfile.c
@@ -0,0 +1,206 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ */
+
+struct KLimitFile;
+#define KFILE_IMPL struct KLimitFile
+
+#include <kfs/extern.h>
+#include <klib/log.h>
+#include <klib/debug.h>
+#include <klib/rc.h>
+#include <kfs/file.h>
+#include <kfs/limitfile.h>
+#include <sysalloc.h>
+
+#include <assert.h>
+#include <stdlib.h>
+
+#include <kfs/impl.h>
+
+/*-----------------------------------------------------------------------
+ * KLimitFile
+ */
+typedef struct KLimitFile KLimitFile;
+struct KLimitFile
+{
+ KFile dad;
+ uint64_t block_size;
+ KFile * original;
+};
+
+/* Destroy
+ */
+static
+rc_t CC KLimitFileDestroy (KLimitFile *self)
+{
+ rc_t rc = KFileRelease ( self -> original );
+ if ( rc == 0 )
+ {
+ self -> original = NULL;
+ free ( self );
+ }
+ return rc;
+}
+
+/* GetSysFile
+ */
+
+static
+struct KSysFile * CC KLimitFileGetSysFile (const KLimitFile *self, uint64_t *offset)
+{
+ return KFileGetSysFile ( self -> original, offset );
+}
+
+/* RandomAccess
+ */
+static
+rc_t CC KLimitFileRandomAccess (const KLimitFile *self)
+{
+ return KFileRandomAccess (self->original);
+}
+
+/* Size
+ */
+static
+rc_t CC KLimitFileSize (const KLimitFile *self, uint64_t *size)
+{
+ return KFileSize ( self -> original, size );
+}
+
+/* SetSize
+ */
+static
+rc_t CC KLimitFileSetSize (KLimitFile *self, uint64_t size)
+{
+ return KFileSetSize (self->original, size);
+}
+
+/* Read
+ */
+static
+rc_t CC KLimitFileRead (const KLimitFile *self,
+ uint64_t pos,
+ void *buffer,
+ size_t bsize,
+ size_t *num_read)
+{
+ uint64_t end = pos + bsize;
+ uint64_t limit = ( pos + self -> block_size ) & ~ ( self -> block_size - 1 );
+ if ( end > limit )
+ end = limit;
+ return KFileRead ( self -> original, pos, buffer, ( size_t ) ( end - pos ), num_read );
+}
+
+/* Write
+ */
+static
+rc_t CC KLimitFileWrite (KLimitFile *self, uint64_t pos,
+ const void *buffer, size_t bsize,
+ size_t *num_writ)
+{
+ uint64_t end = pos + bsize;
+ uint64_t limit = ( pos + self -> block_size ) & ~ ( self -> block_size - 1 );
+ if ( end > limit )
+ end = limit;
+ return KFileWrite ( self -> original, pos, buffer, ( size_t ) ( end - pos ), num_writ );
+}
+
+/* Type
+ */
+static
+uint32_t CC KLimitFileType (const KLimitFile *self)
+{
+ return KFileType (self->original);
+}
+
+
+static const KFile_vt_v1 vtKLimitFile =
+{
+ /* version */
+ 1, 1,
+
+ /* 1.0 */
+ KLimitFileDestroy,
+ KLimitFileGetSysFile,
+ KLimitFileRandomAccess,
+ KLimitFileSize,
+ KLimitFileSetSize,
+ KLimitFileRead,
+ KLimitFileWrite,
+
+ /* 1.1 */
+ KLimitFileType
+};
+
+/* ----------------------------------------------------------------------
+ * KLimitFileMake
+ * create a new file object
+ */
+
+LIB_EXPORT rc_t CC KFileMakeLimitFile ( KFile ** pself, const KFile * original, size_t block_size )
+{
+ rc_t rc;
+
+ if ( pself == NULL )
+ rc = RC ( rcFS, rcFile, rcAllocating, rcParam, rcNull );
+ else
+ {
+ if ( original == NULL )
+ rc = RC ( rcFS, rcFile, rcAllocating, rcParam, rcNull );
+ else if ( ( ( block_size - 1 ) & block_size ) != 0 )
+ rc = RC ( rcFS, rcFile, rcAllocating, rcParam, rcInvalid );
+ else
+ {
+ KLimitFile * obj = calloc ( 1, sizeof * obj );
+ if ( obj == NULL )
+ rc = RC ( rcFS, rcFile, rcAllocating, rcMemory, rcExhausted );
+ else
+ {
+ rc = KFileInit ( & obj -> dad, (const KFile_vt*) & vtKLimitFile,
+ "KLimitFile", "no-name",
+ original->read_enabled,
+ original->write_enabled
+ );
+ if ( rc == 0 )
+ {
+ rc = KFileAddRef ( original );
+ if ( rc == 0 )
+ {
+ obj -> original = ( KFile * ) original;
+ obj -> block_size = block_size;
+ * pself = & obj -> dad;
+ return 0;
+ }
+ }
+
+ free ( obj );
+ }
+ }
+
+ * pself = NULL;
+ }
+
+ return rc;
+}
diff --git a/libs/kfs/unix/sysdir.c b/libs/kfs/unix/sysdir.c
index ab0c815..320215f 100644
--- a/libs/kfs/unix/sysdir.c
+++ b/libs/kfs/unix/sysdir.c
@@ -71,6 +71,7 @@ struct KSysDirListing;
#include <dirent.h>
#include <sys/stat.h>
#include <sys/types.h>
+#include <sys/statvfs.h> /* statvfs */
#include <utime.h>
#include <assert.h>
@@ -2412,3 +2413,26 @@ LIB_EXPORT rc_t CC KDirectoryNativeDir_v1 ( KDirectory_v1 **dirp )
return rc;
}
+
+LIB_EXPORT rc_t CC KDirectoryGetDiskFreeSpace_v1 ( const KDirectory * self,
+ uint64_t * free_bytes_available, uint64_t * total_number_of_bytes )
+{
+ if ( self == NULL )
+ return RC ( rcFS, rcDirectory, rcAccessing, rcSelf, rcNull );
+ else {
+ KSysDir_v1 * dir = ( KSysDir_v1 * ) self;
+ struct statvfs buf;
+ memset ( & buf, 0, sizeof buf );
+ if ( statvfs ( dir -> path, & buf) == 0 ) {
+ if ( free_bytes_available != NULL ) {
+ * free_bytes_available = buf . f_bavail * buf . f_frsize;
+ }
+ if ( total_number_of_bytes != NULL ) {
+ * total_number_of_bytes = buf . f_blocks * buf . f_frsize;
+ }
+ return 0;
+ }
+
+ return RC ( rcFS, rcDirectory, rcAccessing, rcError, rcUnknown );
+ }
+}
diff --git a/libs/klib/log.c b/libs/klib/log.c
index db696af..9d4b38c 100644
--- a/libs/klib/log.c
+++ b/libs/klib/log.c
@@ -353,10 +353,8 @@ rc_t logsubstituteparams ( const char* msg, uint32_t argc, const wrt_nvp_t argv[
}
}
/* substitute param value */
- for(value = arg->value; *value != 0; value++, sz++) {
- if( sz < bsize ) {
+ for(value = arg->value; *value != 0 && sz < bsize; value++, sz++) {
buffer[sz] = *value;
- }
}
/* compensate for outer loop's increment */
--sz;
@@ -469,7 +467,24 @@ static
rc_t prep_v_args( uint32_t* argc, wrt_nvp_t argv[], size_t max_argc,
char* pbuffer, size_t pbsize, const char* fmt, va_list args )
{
- rc_t rc = string_vprintf ( pbuffer, pbsize, NULL, fmt, args );
+ size_t num_writ = 0;
+ rc_t rc = string_vprintf ( pbuffer, pbsize, & num_writ, fmt, args );
+ if ( rc == SILENT_RC ( rcText, rcString, rcConverting, rcBuffer,
+ rcInsufficient ) )
+ {
+ size_t pos = num_writ;
+ char truncated [] = "... [ truncated ]";
+ size_t required = num_writ + sizeof truncated;
+ if ( required > pbsize ) {
+ assert ( pbsize > sizeof truncated );
+ pos = pbsize - sizeof truncated;
+ }
+ {
+ size_t c = string_copy_measure ( pbuffer + pos, pbsize, truncated );
+ assert ( c + 1 == sizeof truncated );
+ rc = 0;
+ }
+ }
if ( rc == 0 )
{
/* tokenize the parameters into name/value pairs */
@@ -625,6 +640,7 @@ rc_t log_print( KFmtHandler* formatter, const KLogFmtFlags flags, KWrtHandler* w
rc = prep_v_args(&argc, argv, sizeof(argv)/sizeof(argv[0]) - 1, pbuffer, sizeof(pbuffer), fmt, args);
}
if( rc == 0 && (flags & klogFmtMessage) ) {
+ int retries = 0;
if( msg == NULL || msg[0] == '\0' ) {
msg = "empty log message";
}
@@ -649,7 +665,9 @@ rc_t log_print( KFmtHandler* formatter, const KLogFmtFlags flags, KWrtHandler* w
}
break;
}
- } while(rc == 0);
+ if ( retries ++ > 9 ) /* something is wrong: too many retries */
+ break;
+ } while(rc != 0);
}
}
if( rc != 0 ) {
diff --git a/libs/klib/release-vers.h b/libs/klib/release-vers.h
index 79ec52d..a94a3ef 100644
--- a/libs/klib/release-vers.h
+++ b/libs/klib/release-vers.h
@@ -28,7 +28,7 @@
/* Version of current SRA Toolkit Release */
-#define RELEASE_VERS 0x02080001
+#define RELEASE_VERS 0x02080002
/* Type of Version of current SRA Toolkit Release is one of:
diff --git a/libs/klib/time.c b/libs/klib/time.c
index c9d8f99..275856c 100644
--- a/libs/klib/time.c
+++ b/libs/klib/time.c
@@ -1,3 +1,31 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+
#include <klib/time.h> /* KSleep */
+
LIB_EXPORT rc_t CC KSleep(uint32_t seconds) { return KSleepMs(seconds * 1000); }
diff --git a/libs/klib/unix/systime.c b/libs/klib/unix/systime.c
index 9bef887..a333fb7 100644
--- a/libs/klib/unix/systime.c
+++ b/libs/klib/unix/systime.c
@@ -30,6 +30,7 @@
#include <unistd.h>
#include <stdlib.h>
+#include <string.h> /* memset */
#include <stdio.h>
#include <assert.h>
#include <time.h>
@@ -53,9 +54,9 @@ LIB_EXPORT KTime_t CC KTimeStamp ( void )
LIB_EXPORT KTimeMs_t CC KTimeMsStamp ( void )
{
- struct timeval tm;
+ struct timeval tm;
gettimeofday( &tm, NULL );
- return ( ( tm.tv_sec * 1000 ) + ( tm.tv_usec / 1000 ) );
+ return ( ( tm.tv_sec * 1000 ) + ( tm.tv_usec / 1000 ) );
}
/*--------------------------------------------------------------------------
@@ -127,7 +128,7 @@ LIB_EXPORT KTime_t CC KTimeMakeTime ( const KTime *self )
{
struct tm t;
- assert ( self -> year >= 1900 );
+ assert ( self -> year >= 1900 ); // TODO
t . tm_year = self -> year - 1900;
t . tm_mon = self -> month;
t . tm_mday = self -> day + 1;
@@ -141,22 +142,73 @@ LIB_EXPORT KTime_t CC KTimeMakeTime ( const KTime *self )
t . tm_isdst = self -> dst;
ts = mktime ( &t );
+ ts -= timezone;
}
return ts;
}
+LIB_EXPORT const KTime* CC KTimeFromIso8601 ( KTime *kt, const char * s,
+ size_t size )
+{
+ struct tm t;
+
+ const char * c = NULL;
+
+ if ( kt == NULL || s == NULL )
+ return NULL;
+
+ memset ( & t, 0, sizeof t );
+
+ if ( size == 19 )
+ c = strptime ( s, "%Y-%m-%dT%H:%M:%S", & t );
+ else if ( size == 20 )
+ c = strptime ( s, "%Y-%m-%dT%H:%M:%S%z", & t );
+ else
+ return NULL;
+
+ if ( c != NULL && c - s != size )
+ return NULL;
+
+ memset ( kt, 0, sizeof * kt );
+ KTimeMake ( kt, & t );
+
+ return kt;
+}
+
+
+/* Iso8601
+ * populate "s" from "ks" according to ISO-8601:
+ * YYYY-MM-DDThh:mm:ssTZD
+ */
+KLIB_EXTERN size_t CC KTimeIso8601 ( KTime_t ts, char * s, size_t size ) {
+ const KTime * r = NULL;
+ KTime now;
+
+ time_t unix_time = ( time_t ) ts;
+ struct tm t;
+
+ if ( ts == 0 || s == NULL || size == 0 )
+ return 0;
+
+ r = KTimeGlobal ( & now, ts );
+ if ( r == NULL )
+ return 0;
+
+ gmtime_r ( & unix_time, & t );
+ return strftime ( s, size, "%FT%TZ", & t );
+}
+
+
LIB_EXPORT rc_t CC KSleepMs(uint32_t milliseconds) {
struct timespec time;
time.tv_sec = (milliseconds / 1000);
time.tv_nsec = (milliseconds % 1000) * 1000 * 1000;
- if (nanosleep(&time, NULL)) {
+ if (nanosleep(&time, NULL))
return 0;
- }
- else {
+ else
return RC(rcRuntime, rcTimeout, rcWaiting, rcTimeout, rcInterrupted);
- }
}
diff --git a/libs/klib/vector_namelist.c b/libs/klib/vector_namelist.c
index ae6daa4..05572b5 100644
--- a/libs/klib/vector_namelist.c
+++ b/libs/klib/vector_namelist.c
@@ -444,6 +444,65 @@ LIB_EXPORT rc_t CC VNamelistSplitStr ( VNamelist * list, const char * str, const
}
+LIB_EXPORT rc_t CC VNamelistFromKNamelist ( VNamelist ** list, const KNamelist * src )
+{
+ rc_t rc = 0;
+ if ( list == NULL )
+ rc = RC ( rcCont, rcNamelist, rcParsing, rcSelf, rcNull );
+ else
+ {
+ *list = NULL;
+ if ( src == NULL )
+ rc = RC ( rcCont, rcNamelist, rcParsing, rcParam, rcNull );
+ else
+ {
+ uint32_t count;
+ rc = KNamelistCount ( src, &count );
+ if ( rc == 0 )
+ {
+ if ( count == 0 )
+ rc = RC ( rcCont, rcNamelist, rcParsing, rcParam, rcEmpty );
+ else
+ {
+ rc = VNamelistMake( list, count );
+ if ( rc == 0 )
+ {
+ uint32_t idx;
+ for ( idx = 0; rc == 0 && idx < count; ++idx )
+ {
+ const char * s = NULL;
+ rc = KNamelistGet ( src, idx, &s );
+ if ( rc == 0 && s != NULL )
+ {
+ rc = VNamelistAppend ( *list, s );
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+
+LIB_EXPORT rc_t CC CopyVNamelist ( VNamelist ** list, const VNamelist * src )
+{
+ rc_t rc = 0;
+ if ( list == NULL )
+ rc = RC ( rcCont, rcNamelist, rcParsing, rcSelf, rcNull );
+ else
+ {
+ const KNamelist * casted;
+ *list = NULL;
+ rc = VNamelistToConstNamelist ( src, &casted );
+ if ( rc == 0 )
+ rc = VNamelistFromKNamelist ( list, casted );
+ }
+ return rc;
+}
+
+
LIB_EXPORT rc_t CC VNamelistFromString ( VNamelist ** list, const String * str, const uint32_t delim )
{
rc_t rc = VNamelistMake( list, 10 );
diff --git a/libs/kns/Makefile b/libs/kns/Makefile
index e6f3067..ac55877 100644
--- a/libs/kns/Makefile
+++ b/libs/kns/Makefile
@@ -75,6 +75,7 @@ clean: stdclean
$(ILIBDIR)/libkns: $(addprefix $(ILIBDIR)/libkns.,$(ILIBEXT))
KNS_NO_HTTP_SRC = \
+ stream-from-buffer \
kns_manager-ext \
manager \
buffered-stream \
diff --git a/libs/kns/http-client.c b/libs/kns/http-client.c
index b022e1b..730a6d6 100644
--- a/libs/kns/http-client.c
+++ b/libs/kns/http-client.c
@@ -139,6 +139,8 @@ struct KClientHttp
bool reliable;
bool tls;
+
+ bool close_connection;
};
@@ -180,6 +182,14 @@ rc_t KClientHttpClear ( KClientHttp *self )
static
rc_t KClientHttpWhack ( KClientHttp * self )
{
+ if ( self -> close_connection )
+ {
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ),
+ ("*** closing connection ***\n"));
+ KClientHttpClose ( self );
+ self -> close_connection = false;
+ }
+
KClientHttpClear ( self );
KDataBufferWhack ( & self -> block_buffer );
@@ -284,9 +294,6 @@ static
rc_t KClientHttpGetLine ( KClientHttp *self, struct timeout_t *tm );
static
-rc_t KClientHttpGetStatusLine ( KClientHttp *self, timeout_t *tm, String *msg, uint32_t *status, ver_t *version );
-
-static
rc_t KClientHttpProxyConnect ( KClientHttp * self, const String * hostname, uint32_t port, KSocket * sock,
const String * phostname, uint32_t pport )
{
@@ -400,14 +407,17 @@ rc_t KClientHttpOpen ( KClientHttp * self, const String * aHostname, uint32_t aP
STATUS ( STAT_QA, "%s - opening socket to %S:%u\n", __func__, aHostname, aPort );
- assert ( self != NULL );
+ assert ( self );
mgr = self -> mgr;
+ assert ( mgr );
KEndPointArgsIteratorMake ( & it, mgr, aHostname, aPort );
while ( KEndPointArgsIteratorNext
( & it, & hostname, & port, & proxy_default_port, & proxy_ep ) )
{
rc = KNSManagerInitDNSEndpoint ( mgr, & self -> ep, hostname, port );
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ),
+ ( "KNSManagerInitDNSEndpoint(%S:%d)=%R\n", hostname, port, rc ) );
if ( rc == 0 )
{
self -> proxy_default_port = proxy_default_port;
@@ -436,10 +446,17 @@ rc_t KClientHttpOpen ( KClientHttp * self, const String * aHostname, uint32_t aP
}
}
+ if ( rc != 0 ) {
+ if ( KNSManagerLogNcbiVdbNetError ( mgr ) )
+ PLOGERR ( klogSys, ( klogSys, rc, "Failed to Make Connection "
+ "in KClientHttpOpen to '$(host):$(port)",
+ "host=%S,port=%hd", aHostname, aPort ) );
+ }
/* if the connection is open */
- if ( rc == 0 )
+ else
{
- STATUS ( STAT_USR, "%s - connected to %S\n", __func__, hostname );
+ STATUS ( STAT_USR, "%s - connected to %S (%s)\n", __func__, hostname,
+ self -> ep . ip_address );
if ( self -> tls )
{
KTLSStream * tls_stream;
@@ -449,14 +466,23 @@ rc_t KClientHttpOpen ( KClientHttp * self, const String * aHostname, uint32_t aP
if ( rc != 0 )
{
- if ( ! proxy_ep )
- DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ), ( "Failed to create TLS stream for '%S'\n", aHostname ) );
+ if ( ! proxy_ep ) {
+ if ( KNSManagerLogNcbiVdbNetError ( mgr ) )
+ PLOGERR ( klogSys, ( klogSys, rc,
+ "Failed to create TLS stream for '$(host)' ($(ip))",
+ "host=%S,ip=%s", aHostname, self -> ep . ip_address
+ ) );
+ else
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_TLS ),
+ ( "Failed to create TLS stream for '%S' (%s)\n",
+ aHostname, self -> ep . ip_address ) );
+ }
else
{
STATUS ( STAT_PRG, "%s - retrying TLS wrapper on socket with proxy hostname\n", __func__ );
rc = KNSManagerMakeTLSStream ( mgr, & tls_stream, sock, hostname );
if ( rc != 0 )
- DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ), ( "Failed to create TLS stream for '%S'\n", hostname ) );
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_TLS ), ( "Failed to create TLS stream for '%S'\n", hostname ) );
}
}
@@ -816,6 +842,26 @@ LIB_EXPORT rc_t CC KClientHttpRelease ( const KClientHttp *self )
return 0;
}
+
+LIB_EXPORT rc_t CC KClientHttpRead ( const KClientHttp *self,
+ void *buffer, size_t bsize, size_t *num_read )
+{
+ if ( self == NULL )
+ return RC ( rcNS, rcNoTarg, rcReading, rcSelf, rcNull );
+
+ return KStreamRead ( self -> sock, buffer, bsize, num_read );
+}
+
+LIB_EXPORT rc_t CC KClientHttpWriteAll ( const KClientHttp *self,
+ const void *buffer, size_t size, size_t *num_writ )
+{
+ if ( self == NULL )
+ return RC ( rcNS, rcNoTarg, rcReading, rcSelf, rcNull );
+
+ return KStreamWriteAll ( self -> sock, buffer, size, num_writ );
+}
+
+
/* Communication Methods
* Read in the http response and return 1 char at a time
*/
@@ -1132,7 +1178,6 @@ rc_t KClientHttpReplaceHeader
}
/* Capture each header line to add to BSTree */
-static
rc_t KClientHttpGetHeaderLine ( KClientHttp *self, timeout_t *tm, BSTree *hdrs,
bool * blank, bool * len_zero, bool * close_connection )
{
@@ -1141,15 +1186,19 @@ rc_t KClientHttpGetHeaderLine ( KClientHttp *self, timeout_t *tm, BSTree *hdrs,
if ( rc == 0 )
{
/* blank = empty line_buffer = separation between headers and body of response */
- if ( self -> line_valid == 0 )
+ if ( self -> line_valid == 0 ) {
* blank = true;
+
+ /* print an empty string to separate response headers */
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_HTTP ), ( "\n" ) );
+ }
else
{
char * sep;
const char * buffer = ( char * ) self -> line_buffer . base;
const char * end = buffer + self -> line_valid;
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS), ("HTTP receive '%s'\n", buffer));
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_HTTP), ("HTTP receive '%s'\n", buffer));
/* find the separation between name: value */
sep = string_chr ( buffer, end - buffer, ':' );
@@ -1195,7 +1244,7 @@ rc_t KClientHttpGetHeaderLine ( KClientHttp *self, timeout_t *tm, BSTree *hdrs,
if ( strcase_cmp ( name . addr, name . size, "Connection", name . size, ( uint32_t ) name . size ) == 0 &&
strcase_cmp ( value . addr, value . size, "close", value . size, ( uint32_t ) value . size ) == 0 )
{
- DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ),
("*** seen connection close ***\n"));
* close_connection = true;
}
@@ -1258,7 +1307,6 @@ rc_t KClientHttpFindHeader ( const BSTree *hdrs, const char *_name, char *buffer
return rc;
}
-static
rc_t KClientHttpGetStatusLine ( KClientHttp *self, timeout_t *tm, String *msg, uint32_t *status, ver_t *version )
{
/* First time reading the response */
@@ -1277,7 +1325,7 @@ rc_t KClientHttpGetStatusLine ( KClientHttp *self, timeout_t *tm, String *msg, u
const char * buffer = ( char * ) self -> line_buffer . base;
const char * end = buffer + self -> line_valid;
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS), ("HTTP receive '%s'\n", buffer));
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_HTTP), ("HTTP receive '%s'\n", buffer));
/* Detect protocol
expect HTTP/1.[01]<sp><digit>+<sp><msg>\r\n */
@@ -1293,7 +1341,7 @@ rc_t KClientHttpGetStatusLine ( KClientHttp *self, timeout_t *tm, String *msg, u
if ( strcase_cmp ( "http", 4, buffer, sep - buffer, 4 ) != 0 )
{
rc = RC ( rcNS, rcNoTarg, rcParsing, rcNoObj, rcUnsupported );
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS), ("%s: protocol given as '%.*s'\n", __func__, ( uint32_t ) ( sep - buffer ), buffer ));
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_HTTP), ("%s: protocol given as '%.*s'\n", __func__, ( uint32_t ) ( sep - buffer ), buffer ));
}
else
{
@@ -1703,22 +1751,19 @@ struct KClientHttpResult
KRefcount refcount;
bool len_zero;
- bool close_connection;
};
static
rc_t KClientHttpResultWhack ( KClientHttpResult * self )
{
BSTreeWhack ( & self -> hdrs, KHttpHeaderWhack, NULL );
- if ( self -> close_connection )
- {
- DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
- ("*** closing connection ***\n"));
- KClientHttpClose ( self -> http );
- }
+
KClientHttpRelease ( self -> http );
+
KRefcountWhack ( & self -> refcount, "KClientHttpResult" );
+
free ( self );
+
return 0;
}
@@ -1738,7 +1783,7 @@ rc_t KClientHttpSendReceiveMsg ( KClientHttp *self, KClientHttpResult **rslt,
if ( KNSManagerIsVerbose ( self -> mgr ) )
KOutMsg ( "KClientHttpSendReceiveMsg: '%.*s'\n", len, buffer );
#endif
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS),
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_HTTP),
("HTTP send '%S' '%.*s'\n\n", &self->hostname, len, buffer));
/* reopen connection if NULL */
@@ -1833,7 +1878,7 @@ rc_t KClientHttpSendReceiveMsg ( KClientHttp *self, KClientHttpResult **rslt,
for ( blank = false; ! blank && rc == 0; )
{
rc = KClientHttpGetHeaderLine ( self, & tm, & result -> hdrs,
- & blank, & result -> len_zero, & result -> close_connection );
+ & blank, & result -> len_zero, & self -> close_connection );
}
if ( rc == 0 && status != 100 )
@@ -2361,8 +2406,11 @@ LIB_EXPORT rc_t CC KClientHttpResultGetInputStream ( KClientHttpResult *self, KS
return KClientHttpStreamMake ( self -> http, s, "KClientHttpStream", content_length, false );
/* detect connection: close or pre-HTTP/1.1 dynamic content */
- if ( self -> close_connection || self -> version < 0x01010000 )
+ if ( self -> http -> close_connection ||
+ self -> version < 0x01010000 )
+ {
return KClientHttpStreamMake ( self -> http, s, "KClientHttpStream", 0, true );
+ }
#if _DEBUGGING
KOutMsg ( "HTTP/%.2V %03u %S\n", self -> version, self -> status, & self -> msg );
@@ -2553,6 +2601,13 @@ LIB_EXPORT rc_t CC KClientHttpMakeRequest ( const KClientHttp *self,
return rc;
}
+void KClientHttpGetEndpoint ( const KClientHttp * self, KEndPoint * ep ) {
+ assert ( ep );
+ memset ( ep, 0, sizeof * ep );
+ if ( self != NULL )
+ * ep = self -> ep;
+}
+
/* MakeRequest
* create a request that can be used to contact HTTP server
*
@@ -3260,7 +3315,7 @@ rc_t KClientHttpRequestSendReceiveNoBodyInt ( KClientHttpRequest *self, KClientH
default:
- if ( ! rslt -> len_zero || rslt -> close_connection )
+ if ( ! rslt -> len_zero || self -> http -> close_connection )
{
/* the connection is no good */
KClientHttpClose ( self -> http );
@@ -3440,7 +3495,7 @@ rc_t CC KClientHttpRequestPOST_Int ( KClientHttpRequest *self, KClientHttpResult
default:
- if ( ! rslt -> len_zero || rslt -> close_connection )
+ if ( ! rslt -> len_zero || self -> http -> close_connection )
{
/* the connection is no good */
KClientHttpClose ( self -> http );
@@ -3480,7 +3535,7 @@ static bool GovSiteByHttp ( const char * path ) {
strcase_cmp ( path, size, http . addr, size, size ) == 0 )
{
EUrlParseState state = eUPSBegin;
- int i = 0;
+ unsigned i = 0;
for ( i = 7; i < path_size && state != eUPSDone; ++i ) {
switch ( state ) {
case eUPSBegin:
diff --git a/libs/kns/http-file.c b/libs/kns/http-file.c
index 33f0852..0e363bf 100644
--- a/libs/kns/http-file.c
+++ b/libs/kns/http-file.c
@@ -378,7 +378,8 @@ rc_t CC KHttpFileTimedRead ( const KHttpFile *self,
if ( rc == 0 )
{
- DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_HTTP ), ( "KHttpFileTimedRead(pos=%lu)\n", pos ) );
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_HTTP ),
+ ( "KHttpFileTimedRead(pos=%lu,size=%zu)...\n", pos, bsize ) );
/* loop using existing KClientHttp object */
while ( rc == 0 )
@@ -406,6 +407,10 @@ rc_t CC KHttpFileTimedRead ( const KHttpFile *self,
}
if ( ! KHttpRetrierWait ( & retrier, http_status ) )
{
+ assert ( num_read );
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_HTTP ),
+ ( "...KHttpFileTimedRead(pos=%lu,size=%zu)=%zu\n\n",
+ pos, bsize, * num_read ) );
break;
}
rc = KClientHttpReopen ( self -> http );
@@ -424,9 +429,18 @@ static
rc_t CC KHttpFileRead ( const KHttpFile *self, uint64_t pos,
void *buffer, size_t bsize, size_t *num_read )
{
+ rc_t rc = 0;
struct timeout_t tm;
TimeoutInit ( & tm, self -> kns -> http_read_timeout );
- return KHttpFileTimedRead ( self, pos, buffer, bsize, num_read, & tm );
+ rc = KHttpFileTimedRead ( self, pos, buffer, bsize, num_read, & tm );
+ if ( rc != 0 && KNSManagerLogNcbiVdbNetError ( self -> kns ) ) {
+ KEndPoint ep;
+ KClientHttpGetEndpoint ( self -> http, & ep );
+ PLOGERR ( klogErr, ( klogErr, rc,
+ "Failed to KHttpFileRead('$(path)' ($(ip)))", "path=%s,ip=%s",
+ self -> url_buffer . base, ep . ip_address ) );
+ }
+ return rc;
}
static
@@ -587,10 +601,43 @@ static rc_t KNSManagerVMakeHttpFileInt ( const KNSManager *self,
return 0;
}
}
+ else {
+ KEndPoint ep;
+ KClientHttpGetEndpoint ( http, & ep );
+ if ( KNSManagerLogNcbiVdbNetError ( self ) ) {
+ bool print = true;
+ String vdbcache;
+ CONST_STRING ( & vdbcache, ".vdbcache" );
+ if ( buf -> elem_count > vdbcache . size ) {
+ String ext;
+ StringInit ( & ext, ( char * ) buf -> base
+ + buf -> elem_count - vdbcache . size - 1,
+ vdbcache . size, vdbcache . size );
+ if ( ext . addr [ ext . size ] == '\0' &&
+ StringEqual ( & vdbcache, & ext ) )
+ {
+ print = false;
+ }
+ }
+ if ( print ) {
+ assert ( buf );
+ PLOGERR ( klogErr,
+ ( klogErr, rc,
+ "Failed to KNSManagerVMakeHttpFileInt('$(path)' ($(ip)))",
+ "path=%.*s,ip=%s",
+ ( int ) buf -> elem_count, buf -> base, ep . ip_address
+ ) );
+ }
+ }
+ else
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS_HTTP ),
+ ( "Failed to KNSManagerVMakeHttpFileInt('%.*s' (%s))\n",
+ ( int ) buf -> elem_count, buf -> base,
+ ep . ip_address ) );
+ }
}
}
}
-
KClientHttpRelease ( http );
}
}
@@ -609,6 +656,7 @@ static rc_t KNSManagerVMakeHttpFileInt ( const KNSManager *self,
return rc;
}
+/******************************************************************************/
LIB_EXPORT rc_t CC KNSManagerMakeHttpFile(const KNSManager *self,
const KFile **file, struct KStream *conn, ver_t vers, const char *url, ...)
diff --git a/libs/kns/http-priv.h b/libs/kns/http-priv.h
index 5019bd6..c7c9a62 100644
--- a/libs/kns/http-priv.h
+++ b/libs/kns/http-priv.h
@@ -59,6 +59,7 @@ struct KFile;
struct KNSManager;
struct KClientHttp;
struct KClientHttpRequest;
+struct KEndPoint;
struct KStream;
struct timeout_t;
struct URLBlock;
@@ -79,16 +80,16 @@ struct KHttpHeader
extern void KHttpHeaderWhack ( BSTNode *n, void *ignore );
extern int64_t CC KHttpHeaderSort ( const BSTNode *na, const BSTNode *nb );
extern int64_t CC KHttpHeaderCmp ( const void *item, const BSTNode *n );
-/*
-extern rc_t KHttpGetHeaderLine ( struct KClientHttp *self, struct timeout_t *tm, BSTree *hdrs, bool *blank, bool *close_connection );
-extern rc_t KHttpGetStatusLine ( struct KClientHttp *self, struct timeout_t *tm, String *msg, uint32_t *status, ver_t *version );
-*/
+
+extern rc_t KClientHttpGetHeaderLine ( struct KClientHttp *self, struct timeout_t *tm, BSTree *hdrs, bool *blank, bool * len_zero, bool *close_connection );
+extern rc_t KClientHttpGetStatusLine ( struct KClientHttp *self, struct timeout_t *tm, String *msg, uint32_t *status, ver_t *version );
+
/* compatibility for existing code */
-/*
+
#define KHttpGetHeaderLine KClientHttpGetHeaderLine
#define KHttpGetStatusLine KClientHttpGetStatusLine
-*/
+
/*--------------------------------------------------------------------------
* KClientHttp
@@ -117,6 +118,9 @@ rc_t KClientHttpMakeRequestInt ( struct KClientHttp const *self,
struct KClientHttpRequest **req, const struct URLBlock *block, const KDataBuffer *buf );
+void KClientHttpGetEndpoint ( const struct KClientHttp * self, struct KEndPoint * ep );
+
+
/* exported private functions
*/
diff --git a/libs/kns/http.c b/libs/kns/http.c
index 5d9547f..fcfcd2b 100644
--- a/libs/kns/http.c
+++ b/libs/kns/http.c
@@ -39,7 +39,6 @@
#undef ERR
#endif
-#include <klib/debug.h> /* DBGMSG */
#include <klib/text.h>
#include <klib/container.h>
#include <klib/out.h>
diff --git a/libs/kns/linux/sysendpoint.c b/libs/kns/linux/sysendpoint.c
index 88f3e77..cf67bf5 100644
--- a/libs/kns/linux/sysendpoint.c
+++ b/libs/kns/linux/sysendpoint.c
@@ -29,7 +29,9 @@
#include <klib/text.h>
#include <klib/printf.h>
#include <klib/rc.h>
+#include <klib/status.h> /* STATUS */
#include <klib/data-buffer.h>
+#include <klib/debug.h> /* DBGMSG */
#include "stream-priv.h"
@@ -113,6 +115,14 @@ rc_t CC KNSManagerInitDNSEndpoint ( struct KNSManager const *self,
);
if ( ghbnr == 0 && remote != NULL )
{
+ struct in_addr ** addr_list
+ = ( struct in_addr ** ) remote -> h_addr_list;
+ string_copy_measure ( ep -> ip_address,
+ sizeof ep -> ip_address,
+ inet_ntoa ( * addr_list [ 0 ] ));
+ STATUS ( STAT_PRG, "%s resolved to %s\n",
+ hostname , ep -> ip_address );
+
ep -> type = epIPV4;
memmove ( & ep -> u . ipv4 . addr, remote -> h_addr_list [ 0 ], sizeof ep -> u . ipv4 . addr );
ep -> u . ipv4 . addr = htonl ( ep -> u . ipv4 . addr );
diff --git a/libs/kns/manager.c b/libs/kns/manager.c
index 8fe8e37..01aa5e2 100644
--- a/libs/kns/manager.c
+++ b/libs/kns/manager.c
@@ -34,6 +34,7 @@
#include <kfg/config.h>
+#include <klib/debug.h> /* DBGMSG */
#include <klib/printf.h>
#include <klib/refcount.h>
#include <klib/rc.h>
@@ -181,6 +182,8 @@ rc_t HttpProxyAddHttpProxyPath ( HttpProxy * self,
self -> http_proxy = proxy;
self -> http_proxy_port = proxy_port;
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ),
+ ( "Added proxy '%S:%d'\n", proxy, proxy_port ) );
if ( ! mgr -> http_proxy_enabled ) {
mgr -> http_proxy_enabled = ( proxy != NULL );
}
@@ -684,13 +687,31 @@ static rc_t CC KNSManagerVSetHTTPProxyPathImpl
break;
#endif
}
- colon = string_rchr ( p, s, ':' );
- if ( colon != NULL )
- {
- char * end;
- const char * port_spec = colon + 1;
- /* it is true that some day we might read symbolic port names... */
- long port_num = strtol ( port_spec, & end, 10 );
+
+ colon = string_chr ( p, s, ':' );
+ if ( colon != NULL ) {
+ char * end = NULL;
+ const char * port_spec = NULL;
+ long port_num = 0;
+
+ int have = colon - p;
+ int remains = s - have;
+ if ( remains > 2 ) {
+ assert ( colon [ 0 ] == ':' );
+ if ( colon [ 1 ] == '/' && colon [ 2 ] == '/' ) {
+ /* strip off the scheme from proxy specification: it is ignored now */
+ psize -= have + 3;
+ p = colon + 3;
+ if ( psize == 0 )
+ return RC ( rcNS, rcMgr, rcUpdating,
+ rcPath, rcInvalid );
+ continue;
+ }
+ }
+
+ port_spec = colon + 1;
+ /* it is true that some day we might read symbolic port names... */
+ port_num = strtol ( port_spec, & end, 10 );
if ( port_num <= 0 || port_num >= 0x10000 ||
( end [ 0 ] != 0 && comma == NULL ) )
rc = RC ( rcNS, rcMgr, rcUpdating, rcPath, rcInvalid );
@@ -938,6 +959,43 @@ LIB_EXPORT bool CC KNSManagerSetHTTPProxyEnabled ( KNSManager * self, bool enabl
}
+static void KNSManagerSetNCBI_VDB_NET ( KNSManager * self, const KConfig * kfg )
+{
+ rc_t rc = 0;
+
+ const KConfigNode * node = NULL;
+
+ if ( self == NULL || kfg == NULL )
+ return;
+
+ rc = KConfigOpenNodeRead ( kfg, & node, "/libs/kns/NCBI_VDB_NET" );
+ if ( rc != 0 ) {
+ self -> NCBI_VDB_NETkfgValueSet = self -> NCBI_VDB_NETkfgValue = false;
+ return;
+ }
+ else {
+ char buffer [ 1 ] = "";
+ size_t num_read = 0;
+ self -> NCBI_VDB_NETkfgValueSet = true;
+ rc = KConfigNodeRead ( node, 0, buffer, sizeof buffer, & num_read, 0 );
+ if ( num_read == 0 )
+ self -> NCBI_VDB_NETkfgValue = false;
+ else switch ( buffer [ 0 ] ) {
+ case '0':
+ case 'f': /* false */
+ self -> NCBI_VDB_NETkfgValue = false;
+ break;
+ default:
+ self -> NCBI_VDB_NETkfgValue = true;
+ break;
+ }
+ }
+
+ KConfigNodeRelease ( node );
+ node = NULL;
+}
+
+
LIB_EXPORT rc_t CC KNSManagerMakeConfig ( KNSManager **mgrp, KConfig* kfg )
{
rc_t rc;
@@ -979,7 +1037,10 @@ LIB_EXPORT rc_t CC KNSManagerMakeConfig ( KNSManager **mgrp, KConfig* kfg )
{
KNSManagerLoadAWS ( mgr, kfg );
KNSManagerHttpProxyInit ( mgr, kfg );
+ KNSManagerSetNCBI_VDB_NET ( mgr, kfg );
+
* mgrp = mgr;
+
return 0;
}
}
@@ -1031,3 +1092,38 @@ LIB_EXPORT rc_t CC KNSManagerGetUserAgent ( const char ** user_agent )
}
return rc;
}
+
+/******************************************************************************/
+
+#define NCBI_VDB_NET 1 /* VDB-3399 : temporarily enable for internal testing */
+
+bool KNSManagerLogNcbiVdbNetError ( const KNSManager * self ) {
+ if ( self == NULL )
+#ifdef NCBI_VDB_NET
+ return true;
+#else
+ return false;
+#endif
+ else {
+ const char * e = getenv ( "NCBI_VDB_NET" );
+ if ( e != NULL ) {
+ if ( e [ 0 ] == '0' ||
+ e [ 0 ] == 'f' ) /* false */
+ {
+ return false;
+ }
+ else
+ return true;
+ }
+ else {
+ if ( self -> NCBI_VDB_NETkfgValueSet )
+ return self -> NCBI_VDB_NETkfgValue;
+ }
+
+#ifdef NCBI_VDB_NET
+ return true;
+#else
+ return false;
+#endif
+ }
+}
diff --git a/libs/kns/mgr-priv.h b/libs/kns/mgr-priv.h
index 3bf479e..469da38 100644
--- a/libs/kns/mgr-priv.h
+++ b/libs/kns/mgr-priv.h
@@ -79,8 +79,15 @@ struct KNSManager
HttpProxy * http_proxy;
bool verbose;
+
+ bool NCBI_VDB_NETkfgValueSet;
+ bool NCBI_VDB_NETkfgValue;
};
+
+bool KNSManagerLogNcbiVdbNetError ( const struct KNSManager * self );
+
+
/* test */
struct KStream;
void KStreamForceSocketClose ( struct KStream const * self );
diff --git a/libs/kns/stream-from-buffer.c b/libs/kns/stream-from-buffer.c
new file mode 100644
index 0000000..6358274
--- /dev/null
+++ b/libs/kns/stream-from-buffer.c
@@ -0,0 +1,192 @@
+/*===========================================================================
+*
+* Public Domain Notice
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#define KSTREAM_IMPL KBufferStream
+typedef struct KBufferStream KBufferStream;
+
+#include <kns/extern.h>
+
+#include <klib/rc.h>
+#include <klib/text.h> /* String */
+#include <kns/impl.h> /* KStream */
+
+#ifndef rcStream
+ #define rcStream rcFile
+#endif
+
+struct KBufferStream {
+ KStream dad;
+ String buffer;
+};
+
+static
+rc_t CC KBufferStreamWhack ( KBufferStream * self )
+{
+ if ( self != NULL ) {
+ memset ( self, 0, sizeof * self );
+ free ( self );
+ }
+
+ return 0;
+}
+
+static
+rc_t CC KBufferStreamRead ( const KBufferStream * self, void * buffer,
+ size_t bsize, size_t * num_read )
+{
+ String * src = NULL;
+
+ size_t dummy = 0;
+ if ( num_read == NULL ) {
+ num_read = & dummy;
+ }
+
+ * num_read = 0;
+
+ assert ( self );
+
+ src = ( String * ) & self -> buffer;
+
+ if ( src -> size == 0 ) {
+ return 0;
+ }
+
+ if ( src -> size < bsize ) {
+ bsize = src -> size;
+ }
+
+ * num_read = string_copy ( buffer, bsize, src -> addr, src -> size );
+
+ src -> addr += * num_read;
+ src -> size -= * num_read;
+ src -> len -= * num_read;
+
+ return 0;
+}
+
+static
+rc_t CC KBufferStreamWrite ( KBufferStream * self, const void * buffer,
+ size_t size, size_t * num_writ )
+{
+ if ( num_writ != NULL ) {
+ * num_writ = 0;
+ }
+
+ return 0;
+}
+
+static
+rc_t CC KBufferStreamTRead ( const KBufferStream * self, void * buffer,
+ size_t bsize, size_t * num_read, struct timeout_t * tm )
+{
+ return KBufferStreamRead ( self, buffer, bsize, num_read );
+}
+
+static
+rc_t CC KBufferStreamTWrite ( KBufferStream * self, const void * buffer,
+ size_t size, size_t * num_writ, struct timeout_t * tm )
+{
+ return KBufferStreamWrite ( self, buffer, size, num_writ );
+}
+
+static KStream_vt_v1 vtKBufferStream = {
+ 1, 1,
+ KBufferStreamWhack,
+ KBufferStreamRead,
+ KBufferStreamWrite,
+ KBufferStreamTRead,
+ KBufferStreamTWrite,
+};
+
+LIB_EXPORT rc_t CC KStreamMakeFromBuffer ( KStream ** self, const char * buffer,
+ size_t size )
+{
+ rc_t rc= 0;
+
+ KBufferStream * obj = NULL;
+
+ if ( self == NULL ) {
+ return RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
+ }
+
+ obj = calloc ( 1, sizeof *obj );
+ if ( obj == NULL ) {
+ return RC ( rcNS, rcStream, rcConstructing, rcMemory, rcExhausted );
+ }
+
+ if ( buffer == NULL ) {
+ size = 0;
+ }
+
+ rc = KStreamInit ( & obj -> dad, ( const KStream_vt* ) & vtKBufferStream,
+ "KBufferStream", "KBufferStream", true, false );
+ if ( rc == 0 ) {
+ StringInit ( & obj -> buffer, buffer, size, size );
+ * self = & obj -> dad;
+ }
+ else {
+ KBufferStreamWhack ( obj );
+ }
+
+ return rc;
+}
+
+/*
+#include "stream.h"
+void test ( void ) {
+ char b[] = "0123456789ABCDEFGHIJKLMNOPQRSTVWXYZabcdefghijklmnopqrstuvwxyz~";
+ int i = 0;
+ for ( i = 0; i < sizeof b; ++i) {
+ b[i] = i;
+ }
+ const KStream * s = NULL;
+puts("TEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEST");
+ rc_t rc = KStreamMakeFromBuffer ( 0, 0 , 0, 0 );
+ assert(rc);
+ rc = KStreamMakeFromBuffer ( & s, 0 , 0, 0 );
+ assert ( ! rc );
+ rc = KStreamRelease ( s );
+ assert ( ! rc );
+ rc = KStreamMakeFromBuffer ( 0 , "X", 0, 0 );
+ assert ( rc );
+ rc = KStreamMakeFromBuffer ( 0 , 0 , b, 0 );
+ assert ( rc );
+ rc = KStreamMakeFromBuffer ( 0 , 0 , 0, sizeof b );
+ assert ( rc );
+ rc = KStreamMakeFromBuffer ( & s, "X", 0, 0 );
+ assert ( ! rc );
+ char c[99] = "";
+ size_t num_read = 0;
+ rc = KStreamRead ( 0, 0, 0, 0 );
+ assert ( rc );
+ rc = KStreamRead ( s, 0, 0, 0 );
+ assert ( rc );
+ rc = KStreamRead ( 0, 0, 0, & num_read );
+ assert ( rc );
+ rc = KStreamRelease ( s );
+ assert ( ! rc );
+}
+*/
diff --git a/libs/kns/tls.c b/libs/kns/tls.c
index 3b50efd..769c0ea 100644
--- a/libs/kns/tls.c
+++ b/libs/kns/tls.c
@@ -28,6 +28,7 @@ struct KTLSStream;
#define KSTREAM_IMPL struct KTLSStream
#include <kns/extern.h>
+#include <kns/endpoint.h> /* KEndPoint */
#include <kns/manager.h>
#include <kns/tls.h>
#include <kns/impl.h>
@@ -127,7 +128,7 @@ static
const char * mbedtls_strerror2 ( int err )
{
static char buffer [ 256 ];
- mbedtls_strerror ( err, buffer, sizeof buffer );
+ vdb_mbedtls_strerror ( err, buffer, sizeof buffer );
return buffer;
}
@@ -136,6 +137,39 @@ const char * mbedtls_strerror2 ( int err )
*/
static
+void ktls_ssl_dbg_print ( void * obj, int level, const char * file, int line, const char * msg )
+{
+ KLogLevel l = klogDebug;
+ switch ( level ) {
+ case 0: /* No debug */
+ l = klogFatal;
+ break;
+ case 1: /* Error */
+ l = klogErr;
+ break;
+ case 2: /* State change */
+ l = klogWarn;
+ break;
+ case 3: /* Informational */
+ l = klogInfo;
+ break;
+ case 4: /* Verbose */
+ default:
+ l = klogDebug;
+ break;
+ }
+
+ if ( file == NULL )
+ file = "mbedtls-file-unknown";
+ if ( msg == NULL )
+ msg = "<missing message>";
+
+ PLOGMSG ( l, ( l, "[$(level)]:$(file):$(line) - $(msg)",
+ "level=%d,file=%s,line=%d,msg=%s",
+ level, file, line, msg ) );
+}
+
+static
rc_t tlsg_seed_rng ( KTLSGlobals *self )
{
int ret;
@@ -144,7 +178,7 @@ rc_t tlsg_seed_rng ( KTLSGlobals *self )
STATUS ( STAT_QA, "Seeding the random number generator\n" );
- ret = mbedtls_ctr_drbg_seed ( &self -> ctr_drbg, mbedtls_entropy_func,
+ ret = vdb_mbedtls_ctr_drbg_seed ( &self -> ctr_drbg, vdb_mbedtls_entropy_func,
&self -> entropy, ( const unsigned char * ) pers, pers_size );
if ( ret != 0 )
@@ -276,7 +310,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self, const KConfig * kfg )
/* these guys take a length, so presumably the string is not NUL terminated.
yet, the first thing they do is see if the NUL is included in the length! */
STATUS ( STAT_GEEK, "Parsing text for node '%s' from CA root certificates\n", cert_name );
- ret = mbedtls_x509_crt_parse ( &self -> cacert,
+ ret = vdb_mbedtls_x509_crt_parse ( &self -> cacert,
( const unsigned char * ) cert_string -> addr, cert_string -> size + 1 );
StringWhack ( cert_string );
@@ -316,7 +350,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self, const KConfig * kfg )
if ( rc2 == 0 )
{
STATUS ( STAT_GEEK, "Parsing text from CA root certificate file '%S'\n", ca_crt_path );
- ret = mbedtls_x509_crt_parse_file ( &self -> cacert, ca_crt_path -> addr );
+ ret = vdb_mbedtls_x509_crt_parse_file ( &self -> cacert, ca_crt_path -> addr );
if ( ret < 0 )
{
PLOGMSG ( klogWarn, ( klogWarn
@@ -337,7 +371,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self, const KConfig * kfg )
if ( num_certs == 0 )
{
STATUS ( STAT_QA, "Parsing text for default CA root certificates\n" );
- ret = mbedtls_x509_crt_parse ( &self -> cacert,
+ ret = vdb_mbedtls_x509_crt_parse ( &self -> cacert,
( const unsigned char * ) ca_crt_ncbi1, sizeof ca_crt_ncbi1 );
if ( ret < 0 )
@@ -354,7 +388,7 @@ rc_t tlsg_init_certs ( KTLSGlobals *self, const KConfig * kfg )
{
num_certs = 1;
- ret = mbedtls_x509_crt_parse ( &self -> cacert,
+ ret = vdb_mbedtls_x509_crt_parse ( &self -> cacert,
( const unsigned char * ) ca_crt_ncbi2, sizeof ca_crt_ncbi2 );
if ( ret >= 0 )
@@ -382,7 +416,7 @@ rc_t tlsg_setup ( KTLSGlobals * self )
STATUS ( STAT_QA, "Configuring SSl defaults\n" );
- ret = mbedtls_ssl_config_defaults ( &self -> config,
+ ret = vdb_mbedtls_ssl_config_defaults ( &self -> config,
MBEDTLS_SSL_IS_CLIENT,
MBEDTLS_SSL_TRANSPORT_STREAM,
MBEDTLS_SSL_PRESET_DEFAULT );
@@ -399,23 +433,61 @@ rc_t tlsg_setup ( KTLSGlobals * self )
return rc;
}
- mbedtls_ssl_conf_authmode( &self -> config, MBEDTLS_SSL_VERIFY_REQUIRED );
- mbedtls_ssl_conf_ca_chain( &self -> config, &self -> cacert, NULL );
- mbedtls_ssl_conf_rng( &self -> config, mbedtls_ctr_drbg_random, &self -> ctr_drbg );
+ vdb_mbedtls_ssl_conf_authmode( &self -> config, MBEDTLS_SSL_VERIFY_REQUIRED );
+ vdb_mbedtls_ssl_conf_ca_chain( &self -> config, &self -> cacert, NULL );
+ vdb_mbedtls_ssl_conf_rng( &self -> config, vdb_mbedtls_ctr_drbg_random, &self -> ctr_drbg );
return 0;
}
+static int set_threshold ( const KConfig * kfg ) {
+ bool set = false;
+
+ int64_t threshold = 0;
+
+ const char * env = NULL;
+
+ rc_t rc = KConfigReadI64 ( kfg, "/tls/NCBI_VDB_TLS", & threshold );
+ if ( rc == 0 )
+ set = true;
+
+ env = getenv ( "NCBI_VDB_TLS" );
+
+ if ( env != NULL ) {
+ int NCBI_VDB_TLS = 0;
+
+ for ( ; * env != '\0'; ++ env ) {
+ char c = * env;
+ if ( c < '0' || c > '9' )
+ break;
+
+ NCBI_VDB_TLS = NCBI_VDB_TLS * 10 + c - '0';
+ set = true;
+ }
+
+ if ( NCBI_VDB_TLS > threshold )
+ threshold = NCBI_VDB_TLS;
+ }
+
+ if ( set )
+ vdb_mbedtls_debug_set_threshold ( threshold );
+
+ return threshold;
+}
+
/* Init
*/
rc_t KTLSGlobalsInit ( KTLSGlobals * tlsg, const KConfig * kfg )
{
rc_t rc;
- mbedtls_x509_crt_init ( &tlsg -> cacert );
- mbedtls_ctr_drbg_init ( &tlsg -> ctr_drbg );
- mbedtls_entropy_init ( &tlsg -> entropy );
- mbedtls_ssl_config_init ( &tlsg -> config );
+ vdb_mbedtls_x509_crt_init ( &tlsg -> cacert );
+ vdb_mbedtls_ctr_drbg_init ( &tlsg -> ctr_drbg );
+ vdb_mbedtls_entropy_init ( &tlsg -> entropy );
+ vdb_mbedtls_ssl_config_init ( &tlsg -> config );
+
+ if ( set_threshold ( kfg ) > 0 )
+ vdb_mbedtls_ssl_conf_dbg ( &tlsg -> config, ktls_ssl_dbg_print, tlsg );
rc = tlsg_seed_rng ( tlsg );
if ( rc == 0 )
@@ -432,10 +504,10 @@ rc_t KTLSGlobalsInit ( KTLSGlobals * tlsg, const KConfig * kfg )
*/
void KTLSGlobalsWhack ( KTLSGlobals * self )
{
- mbedtls_ssl_config_free ( &self -> config );
- mbedtls_entropy_free ( &self -> entropy );
- mbedtls_ctr_drbg_free ( &self -> ctr_drbg );
- mbedtls_x509_crt_free ( &self -> cacert );
+ vdb_mbedtls_ssl_config_free ( &self -> config );
+ vdb_mbedtls_entropy_free ( &self -> entropy );
+ vdb_mbedtls_ctr_drbg_free ( &self -> ctr_drbg );
+ vdb_mbedtls_x509_crt_free ( &self -> cacert );
memset ( self, 0, sizeof * self );
}
@@ -469,8 +541,8 @@ static
void KTLSStreamDestroy ( KTLSStream *self )
{
/* tear down all of the stuff created during Make */
- mbedtls_ssl_close_notify( &self -> ssl ); /* close connection - this might need to be elsewhere */
- mbedtls_ssl_free ( &self -> ssl );
+ vdb_mbedtls_ssl_close_notify( &self -> ssl ); /* close connection - this might need to be elsewhere */
+ vdb_mbedtls_ssl_free ( &self -> ssl );
/* release the ciphertext object */
KStreamRelease ( self -> ciphertext );
@@ -512,7 +584,7 @@ rc_t CC KTLSStreamRead ( const KTLSStream * cself,
while ( 1 )
{
/* read through TLS library */
- ret = mbedtls_ssl_read( &self -> ssl, buffer, bsize );
+ ret = vdb_mbedtls_ssl_read( &self -> ssl, buffer, bsize );
/* no error */
if ( ret >= 0 )
@@ -542,7 +614,7 @@ rc_t CC KTLSStreamRead ( const KTLSStream * cself,
{
/* The ret is anything other than the following 3, then the ssl context becomes
* becomes unusable and should either be freed or call
- * mbedtls_ssl_session_reset () before a new connection; current connection
+ * vdb_mbedtls_ssl_session_reset () before a new connection; current connection
* must be closed
*/
case MBEDTLS_ERR_SSL_WANT_READ:
@@ -554,7 +626,7 @@ rc_t CC KTLSStreamRead ( const KTLSStream * cself,
* is initiating a new connection using the same source port.
* You can either treat that as a connection close and wait
* for the client to resend a ClientHello, or directly
- * continue with \c mbedtls_ssl_handshake() with the same
+ * continue with \c vdb_mbedtls_ssl_handshake() with the same
* context (as it has beeen reset internally). Either way, you
* should make sure this is seen by the application as a new
* connection: application state, if any, should be reset, and
@@ -618,7 +690,7 @@ rc_t CC KTLSStreamWrite ( KTLSStream * self,
* We expect to be called through KStreamWriteAll that will
* avoid the issue above.
*/
- ret = mbedtls_ssl_write ( &self -> ssl, buffer, size );
+ ret = vdb_mbedtls_ssl_write ( &self -> ssl, buffer, size );
/* no error */
if ( ret >= 0 )
@@ -733,7 +805,7 @@ int CC ktls_net_send ( void *ctx, const unsigned char *buf, size_t len )
return ( int ) num_writ;
}
-/* called by mbedtls_ssl_fetch_input */
+/* called by vdb_mbedtls_ssl_fetch_input */
static
int CC ktls_net_recv ( void *ctx, unsigned char *buf, size_t len )
{
@@ -778,7 +850,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self, const String *host )
assert ( self -> mgr != NULL );
tlsg = & self -> mgr -> tlsg;
- ret = mbedtls_ssl_setup( &self -> ssl, &tlsg -> config );
+ ret = vdb_mbedtls_ssl_setup( &self -> ssl, &tlsg -> config );
if ( ret != 0 )
{
rc_t rc = RC ( rcKrypto, rcSocket, rcFormatting, rcEncryption, rcFailed );
@@ -819,7 +891,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self, const String *host )
return rc;
}
- ret = mbedtls_ssl_set_hostname( &self -> ssl, hostz -> addr );
+ ret = vdb_mbedtls_ssl_set_hostname( &self -> ssl, hostz -> addr );
if ( hostz != host )
StringWhack ( hostz );
@@ -837,7 +909,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self, const String *host )
}
- mbedtls_ssl_set_bio( &self -> ssl, ( void * ) self, ktls_net_send, ktls_net_recv, NULL );
+ vdb_mbedtls_ssl_set_bio( &self -> ssl, ( void * ) self, ktls_net_send, ktls_net_recv, NULL );
return 0;
}
@@ -849,7 +921,7 @@ rc_t ktls_handshake ( KTLSStream *self )
STATUS ( STAT_QA, "Performing SSL/TLS handshake...\n" );
- ret = mbedtls_ssl_handshake( &self -> ssl );
+ ret = vdb_mbedtls_ssl_handshake( &self -> ssl );
while ( ret != 0 )
{
if ( ret != MBEDTLS_ERR_SSL_WANT_READ &&
@@ -866,11 +938,11 @@ rc_t ktls_handshake ( KTLSStream *self )
if ( ret == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED )
{
- uint32_t flags = mbedtls_ssl_get_verify_result( &self -> ssl );
+ uint32_t flags = vdb_mbedtls_ssl_get_verify_result( &self -> ssl );
if ( flags != 0 )
{
char buf [ 4096 ];
- mbedtls_x509_crt_verify_info ( buf, sizeof( buf ), " !! ", flags );
+ vdb_mbedtls_x509_crt_verify_info ( buf, sizeof( buf ), " !! ", flags );
PLOGMSG ( klogSys, ( klogSys
, "mbedtls_ssl_get_verify_result returned $(flags) ( $(info) )"
@@ -883,7 +955,7 @@ rc_t ktls_handshake ( KTLSStream *self )
return rc;
}
- ret = mbedtls_ssl_handshake( &self -> ssl );
+ ret = vdb_mbedtls_ssl_handshake( &self -> ssl );
}
return 0;
@@ -919,7 +991,7 @@ rc_t KTLSStreamMake ( KTLSStream ** objp, const KNSManager * mgr, const KSocket
obj -> mgr = mgr;
STATUS ( STAT_PRG, "%s - initializing tls wrapper\n", __func__ );
- mbedtls_ssl_init ( &obj -> ssl );
+ vdb_mbedtls_ssl_init ( &obj -> ssl );
* objp = obj;
return 0;
@@ -983,6 +1055,22 @@ LIB_EXPORT rc_t CC KNSManagerMakeTLSStream ( const KNSManager * self,
*plaintext = ktls;
return 0;
}
+ else {
+ if ( KNSManagerLogNcbiVdbNetError ( self ) ) {
+ KEndPoint ep;
+ rc_t r2 = KSocketGetRemoteEndpoint ( ciphertext,
+ & ep );
+ if ( r2 != 0 )
+ LOGERR ( klogInt, r2
+ , "cannot KSocketGetRemoteEndpoint"
+ );
+ else
+ PLOGERR ( klogSys, ( klogSys, rc,
+ "ktls_handshake failed while accessing '$(ip)'"
+ , "ip=%s", ep . ip_address
+ ) );
+ }
+ }
}
KTLSStreamWhack ( ktls );
@@ -1026,13 +1114,13 @@ LIB_EXPORT rc_t CC KTLSStreamVerifyCACert ( const KTLSStream * self )
rc = RC ( rcKrypto, rcToken, rcValidating, rcSelf, rcNull );
else
{
- uint32_t flags = mbedtls_ssl_get_verify_result( &self -> ssl );
+ uint32_t flags = vdb_mbedtls_ssl_get_verify_result( &self -> ssl );
if ( flags != 0 )
{
char buf [ 4096 ];
rc_t rc = RC ( rcKrypto, rcToken, rcValidating, rcEncryption, rcFailed );
- mbedtls_x509_crt_verify_info ( buf, sizeof( buf ), " ! ", flags );
+ vdb_mbedtls_x509_crt_verify_info ( buf, sizeof( buf ), " ! ", flags );
PLOGERR ( klogSys, ( klogSys, rc
, "mbedtls_ssl_get_verify_result returned $(flags) ( $(info) )"
diff --git a/libs/kns/unix/syssock.c b/libs/kns/unix/syssock.c
index d778f4b..b5b8624 100644
--- a/libs/kns/unix/syssock.c
+++ b/libs/kns/unix/syssock.c
@@ -106,6 +106,8 @@ struct KSocket
struct sockaddr_in6 v6; /* for ipv6 */
} remote_addr;
bool remote_addr_valid;
+
+ char ip_address [ 256 ];
};
LIB_EXPORT rc_t CC KSocketAddRef( const KSocket *self )
@@ -415,6 +417,8 @@ static rc_t KSocketGetEndpoint ( const KSocket * self, KEndPoint * ep, bool remo
rc = RC ( rcNS, rcSocket, rcEvaluating, rcParam, rcNull );
else
{
+ memset ( ep, 0, sizeof * ep );
+
ep -> type = epInvalid;
if ( self == NULL )
@@ -433,6 +437,9 @@ static rc_t KSocketGetEndpoint ( const KSocket * self, KEndPoint * ep, bool remo
rc = RC ( rcNS, rcSocket, rcEvaluating, rcFunction, rcUnsupported );
break;
}
+
+ string_copy_measure ( ep -> ip_address, sizeof ep -> ip_address,
+ self -> ip_address );
}
}
return rc;
@@ -871,6 +878,10 @@ rc_t KSocketConnectIPv4 ( KSocket *self, const KEndPoint *from, const KEndPoint
/* set non-blocking mode */
flag = fcntl ( self -> fd, F_GETFL );
fcntl ( self -> fd, F_SETFL, flag | O_NONBLOCK );
+
+ string_copy_measure ( self -> ip_address,
+ sizeof self -> ip_address, to -> ip_address );
+
return 0;
}
}
diff --git a/libs/ktst/testcase.cpp b/libs/ktst/testcase.cpp
index d9e1493..10adb1e 100644
--- a/libs/ktst/testcase.cpp
+++ b/libs/ktst/testcase.cpp
@@ -33,7 +33,7 @@
using namespace ncbi::NK;
-void TestCase::Init(const char* name) { _name = name; _ec = 0; }
+void TestCase::Init(const std::string& name) { _name = name; _ec = 0; }
void TestCase::report_error(const char* msg, const char* file, int line, bool is_msg, bool isCritical)
{
diff --git a/libs/ktst/testenv.cpp b/libs/ktst/testenv.cpp
index 00fe506..b8bf14c 100644
--- a/libs/ktst/testenv.cpp
+++ b/libs/ktst/testenv.cpp
@@ -394,6 +394,21 @@ rc_t TestEnv::process_args(int argc, char* argv[], ArgsHandler* argsHandler)
pch = strtok(NULL, " ");
}
}
+ else {
+ if ( argc2 >= arg2 ) {
+ arg2 *= 2;
+ char ** tmp = static_cast < char ** > (
+ realloc ( argv2, arg2 * sizeof *argv2 ) );
+ if ( tmp == NULL )
+ return RC (
+ rcApp, rcArgv, rcAccessing, rcMemory, rcExhausted );
+ argv2 = tmp;
+ }
+ argv2 [ argc2 ] = strdup ( argv [ i ] );
+ if ( argv2 [ argc2 ] == NULL )
+ return RC ( rcApp, rcArgv, rcAccessing, rcMemory, rcExhausted );
+ ++ argc2;
+ }
}
if (verbosity == LogLevel::e_undefined)
diff --git a/libs/ncbi-vdb/libncbi-vdb.vers b/libs/ncbi-vdb/libncbi-vdb.vers
index dbe5900..1817afe 100644
--- a/libs/ncbi-vdb/libncbi-vdb.vers
+++ b/libs/ncbi-vdb/libncbi-vdb.vers
@@ -1 +1 @@
-2.8.1
+2.8.2
diff --git a/libs/ngs/BAM_ReadCollection.c b/libs/ngs/BAM_ReadCollection.c
index 4d07a10..23afaaa 100644
--- a/libs/ngs/BAM_ReadCollection.c
+++ b/libs/ngs/BAM_ReadCollection.c
@@ -136,7 +136,7 @@ struct BAM_ReadCollection
unsigned namestart;
unsigned namelen;
unsigned bam_cur; /* current offset in bambuffer */
-
+
uint8_t bambuffer[BAM_BLK_MAX];
};
@@ -168,7 +168,7 @@ static void BAM_ReferenceInfoWhack(BAM_ReferenceInfo *);
static void BAM_ReadCollectionWhack(void *const object, ctx_t ctx) {
BAM_ReadCollection *const self = (BAM_ReadCollection *)object;
-
+
if (self->references) {
BAM_ReferenceInfoWhack(self->references);
free(self->references);
@@ -204,7 +204,7 @@ static NGS_String *BAM_ReferenceGetCommonName(NGS_Reference *const base, ctx_t c
{
BAM_Reference *const self = (BAM_Reference *)base;
char const *name = self->parent->references->ref[self->cur].name;
-
+
return NGS_StringMakeCopy(ctx, name, strlen(name));
}
@@ -225,7 +225,7 @@ static bool BAM_ReferenceGetIsCircular(NGS_Reference const *const base, ctx_t ct
static uint64_t BAM_ReferenceGetLength(NGS_Reference *const base, ctx_t ctx)
{
BAM_Reference *const self = (BAM_Reference *)base;
-
+
return self->parent->references->ref[self->cur].length;
}
@@ -254,7 +254,7 @@ static NGS_Alignment *BAM_ReferenceGetAlignments(NGS_Reference *const base, ctx_
{
BAM_Reference *const self = (BAM_Reference *)base;
HeaderRefInfo const *const ref = &self->parent->references->ref[self->cur];
-
+
if (!wants_primary || !wants_secondary || ref->index == NULL) {
USER_ERROR(xcFunctionUnsupported, "not supported for unindexed BAM");
return 0;
@@ -267,7 +267,7 @@ static uint64_t BAM_ReferenceGetAlignmentCount(NGS_Reference const *const base,
{
BAM_Reference const *const self = (BAM_Reference *)base;
HeaderRefInfo const *const ref = &self->parent->references->ref[self->cur];
-
+
if (!wants_primary || !wants_secondary || ref->index == NULL) {
USER_ERROR(xcFunctionUnsupported, "not supported for unindexed BAM");
return 0;
@@ -285,7 +285,7 @@ static BAM_Record *BAM_GetRecordSliced(NGS_Refcount *const self, ctx_t ctx);
static void BAM_RefIndexSliceWhack(void *const object, ctx_t ctx)
{
BAM_RefIndexSlice *const self = (BAM_RefIndexSlice *)object;
-
+
NGS_RefcountRelease(&self->parent->dad.dad, ctx);
}
@@ -317,7 +317,7 @@ EMPTY_ITERATOR: {
}
uint64_t *chunk;
int const chunks = IndexSlice(ref->index, &chunk, (uint32_t)offset, (uint32_t)(offset + size));
-
+
if (chunks >= 0) {
BAM_RefIndexSlice *const slice = calloc(1, ((uint8_t const *)&((BAM_RefIndexSlice const *)NULL)->chunk[chunks]) - ((uint8_t const *)NULL));
@@ -325,7 +325,7 @@ EMPTY_ITERATOR: {
static NGS_Refcount_vt const vt = {
BAM_RefIndexSliceWhack
};
-
+
NGS_RefcountInit(ctx, &slice->dad, &vt, "BAM_RefIndexSlice", ref->name);
slice->parent = NGS_RefcountDuplicate(&self->parent->dad.dad, ctx);
slice->start = offset;
@@ -336,11 +336,11 @@ EMPTY_ITERATOR: {
memmove(slice->chunk, chunk, chunks * sizeof(*chunk));
free(slice);
-
+
NGS_Alignment *const rslt = BAM_AlignmentMake(ctx, wants_primary, wants_secondary, BAM_GetRecordSliced,
NGS_RefcountDuplicate(&self->dad.dad, ctx),
self->parent->path + self->parent->namestart);
-
+
return rslt;
}
}
@@ -358,14 +358,14 @@ static struct NGS_Pileup *BAM_ReferenceGetPileups(NGS_Reference *const base, ctx
static void BAM_ReferenceWhack(void *const base, ctx_t ctx)
{
BAM_Reference *const self = (BAM_Reference *)base;
-
+
NGS_RefcountRelease(&self->parent->dad.dad, ctx);
}
bool BAM_ReferenceIteratorNext(NGS_Reference *const base, ctx_t ctx)
{
BAM_Reference *const self = (BAM_Reference *)base;
-
+
switch (self->state) {
case 0:
self->state = 1;
@@ -389,7 +389,7 @@ BAM_Reference *BAM_ReferenceMake(BAM_ReadCollection *parent, ctx_t ctx, char con
{
/* NGS_Refcount */
{ BAM_ReferenceWhack },
-
+
/* NGS_Reference */
BAM_ReferenceGetCommonName,
BAM_ReferenceGetCanonicalName,
@@ -402,12 +402,12 @@ BAM_Reference *BAM_ReferenceMake(BAM_ReadCollection *parent, ctx_t ctx, char con
BAM_ReferenceGetAlignmentCount,
BAM_ReferenceGetAlignmentSlice,
BAM_ReferenceGetPileups,
-
+
/* NGS_ReferenceIterator */
BAM_ReferenceIteratorNext,
};
FUNC_ENTRY(ctx, rcSRA, rcCursor, rcConstructing);
-
+
BAM_Reference *const rslt = calloc(1, sizeof(*rslt));
if (rslt) {
NGS_RefcountInit(ctx, &rslt->dad.dad, &vt.dad, "BAM_Reference", name);
@@ -422,7 +422,7 @@ BAM_Reference *BAM_ReferenceMake(BAM_ReadCollection *parent, ctx_t ctx, char con
static NGS_Reference * BAM_ReadCollectionReferences(NGS_ReadCollection *const vp, ctx_t ctx) {
FUNC_ENTRY(ctx, rcSRA, rcFile, rcAccessing);
BAM_ReadCollection *const self = (BAM_ReadCollection *)vp;
-
+
BAM_Reference *const rslt = BAM_ReferenceMake(self, ctx, self->path + self->namestart);
return &rslt->dad;
@@ -433,11 +433,11 @@ static int32_t FindReference(BAM_ReadCollection const *const self, char const na
int32_t i;
int32_t const n = self->references->count;
size_t const nlen = strlen(name);
-
+
for (i = 0; i < n; ++i) {
char const *const fnd = self->references->ref[i].name;
size_t const flen = strlen(fnd);
-
+
if (flen == nlen && strcase_cmp(name, nlen, fnd, nlen, (uint32_t)nlen) == 0)
return i;
}
@@ -447,7 +447,7 @@ static int32_t FindReference(BAM_ReadCollection const *const self, char const na
static NGS_Reference * BAM_ReadCollectionReference(NGS_ReadCollection *const vp, ctx_t ctx, char const spec[]) {
FUNC_ENTRY(ctx, rcSRA, rcFile, rcAccessing);
BAM_ReadCollection *const self = (BAM_ReadCollection *)vp;
-
+
int32_t const fnd = FindReference(self, spec);
if (fnd >= 0) {
BAM_Reference *const rslt = BAM_ReferenceMake(self, ctx, self->path + self->namestart);
@@ -467,13 +467,13 @@ static NGS_Reference * BAM_ReadCollectionReference(NGS_ReadCollection *const vp,
static NGS_Alignment * BAM_ReadCollectionAlignments(NGS_ReadCollection *const vp, ctx_t ctx, bool const wants_primary, bool const wants_secondary) {
FUNC_ENTRY(ctx, rcSRA, rcFile, rcAccessing);
BAM_ReadCollection *const self = (BAM_ReadCollection *)vp;
-
+
if (!wants_primary && !wants_secondary) {
return NGS_AlignmentMakeNull(ctx, self->path + self->namestart, self->namelen);
}
else {
NGS_Alignment *const rslt = BAM_AlignmentMake(ctx, wants_primary, wants_secondary, BAM_GetRecord, NGS_RefcountDuplicate(&self->dad.dad, ctx), self->path + self->namestart);
-
+
return rslt;
}
}
@@ -530,7 +530,7 @@ static struct NGS_Read * BAM_ReadCollectionReadRange(NGS_ReadCollection *const v
static void *Alloc(ctx_t ctx, size_t const size, bool const clear)
{
void *const rslt = clear ? calloc(1, size) : malloc(size);
-
+
if (rslt == NULL) {
SYSTEM_ABORT(xcNoMemory, "allocating %u bytes", size);
}
@@ -544,13 +544,13 @@ static bool FillBuffer(BAM_ReadCollection *const self, ctx_t ctx, uint64_t const
rc_t const rc = KFileRead(self->fp, fpos,
self->iobuffer,
IO_BLK_SIZE, &nread);
-
+
if (rc == 0) {
self->cpos = fpos;
self->fpos = fpos + nread;
self->zs.avail_in = (uInt)nread;
self->zs.next_in = (Bytef *)self->iobuffer;
-
+
return true;
}
@@ -579,7 +579,7 @@ FILL_BUFFER:
if (zrc == Z_STREAM_END) {
/* inflateReset clobbers this value but we want it */
uLong const total_out = self->zs.total_out;
-
+
zrc = inflateReset(&self->zs);
assert(zrc == Z_OK);
self->zs.total_out = total_out;
@@ -599,10 +599,10 @@ static void Seek(BAM_ReadCollection *self, ctx_t ctx, uint64_t const ipos)
{
uint64_t const fpos = (uint64_t)(ipos / BAM_BLK_MAX);
unsigned const bpos = (unsigned)(ipos % BAM_BLK_MAX);
-
+
if (fpos < self->cpos || fpos >= self->fpos || self->zs.next_in == NULL) {
uint64_t const fudg = fpos % IO_BLK_SIZE;
-
+
FillBuffer(self, ctx, fpos - fudg);
if ((unsigned)self->zs.avail_in < fudg)
return;
@@ -620,14 +620,13 @@ static size_t ReadN(BAM_ReadCollection *self, ctx_t ctx, size_t const N, void *D
{
uint8_t *const dst = Dst;
size_t n = 0;
-
+
while (n < N) {
size_t const avail_out = N - n;
size_t const avail_in = self->zs.total_out - self->bam_cur;
if (avail_in) {
size_t const copy = avail_out < avail_in ? avail_out : avail_in;
-
memmove(dst + n, self->bambuffer + self->bam_cur, copy);
self->bam_cur += copy;
n += copy;
@@ -696,16 +695,16 @@ static int32_t ReadI32(BAM_ReadCollection *self, ctx_t ctx)
{
int8_t ch[4];
size_t const n = ReadN(self, ctx, 4, ch);
-
+
if (FAILED())
return 0;
-
+
if (n == 4)
return LE2Int32(ch);
-
+
if (n)
USER_ERROR(xcUnexpected, "reading '%s'; premature end of file", self->path);
-
+
return 0;
}
@@ -804,10 +803,10 @@ static bool ReadBAMRecord(BAM_ReadCollection *const self, ctx_t ctx, BAM_rec out
static void CopyCIGAR(uint32_t dst[], uint32_t const src[], unsigned const count)
{
unsigned i;
-
+
for (i = 0; i < count; ++i) {
uint32_t const value = LE2Int32(src + i);
-
+
dst[i] = value;
}
}
@@ -817,19 +816,19 @@ static void CopySEQ(char dst[], uint8_t const src[], unsigned const count)
static char const tr[16] = "=ACMGRSVTWYHKDBN";
unsigned i;
unsigned const n = count >> 1;
-
+
for (i = 0; i < n; ++i) {
uint8_t const value = src[i];
uint8_t const lo = value & 0x0F;
uint8_t const hi = value >> 4;
-
+
dst[2 * i + 0] = tr[hi];
dst[2 * i + 1] = tr[lo];
}
if (count & 1) {
uint8_t const value = src[n];
uint8_t const hi = value >> 4;
-
+
dst[count - 1] = tr[hi];
}
}
@@ -837,10 +836,10 @@ static void CopySEQ(char dst[], uint8_t const src[], unsigned const count)
static void CopyQUAL(char dst[], uint8_t const src[], unsigned const n)
{
unsigned i;
-
+
for (i = 0; i < n; ++i) {
int const ch = src[i] + 33;
-
+
dst[i] = ch < 0xFF ? ch : -1;
}
}
@@ -850,7 +849,7 @@ BAM_Record *BAM_GetRecord(NGS_Refcount *const object, ctx_t ctx)
FUNC_ENTRY(ctx, rcSRA, rcFile, rcOpening);
BAM_ReadCollection *const self = (BAM_ReadCollection *)object;
BAM_rec raw;
-
+
if (ReadBAMRecord(self, ctx, &raw)) {
BAM_Record *rslt = NULL;
bool self_unmapped = false;
@@ -861,7 +860,7 @@ BAM_Record *BAM_GetRecord(NGS_Refcount *const object, ctx_t ctx)
int32_t const pos = get_pos(&raw);
int32_t const next_refID = get_next_refID(&raw);
int32_t const next_pos = get_next_pos(&raw);
-
+
if ((flag & 0x0004) != 0 || 0 > refID || refID >= self->references->count || pos < 0 || raw_nc == 0)
self_unmapped = true;
if ((flag & 0x0001) == 0 || (flag & 0x0008) != 0 || 0 > next_refID || next_refID >= self->references->count || next_pos < 0)
@@ -889,7 +888,7 @@ BAM_Record *BAM_GetRecord(NGS_Refcount *const object, ctx_t ctx)
rslt->extra = (void const *)&rslt->QNAME[nl];
if (nl == 0)
rslt->QNAME = NULL;
-
+
rslt->TLEN = get_tlen(&raw);
rslt->FLAG = flag;
rslt->MAPQ = get_mq(&raw);
@@ -915,7 +914,7 @@ BAM_Record *BAM_GetRecord(NGS_Refcount *const object, ctx_t ctx)
memmove((void *)rslt->extra, extra, extralen);
CopySEQ((void *)rslt->SEQ, seq, sl);
CopyQUAL((void *)rslt->QUAL, qual, sl);
-
+
if (nl)
memmove((void *)rslt->QNAME, read_name, nl);
@@ -932,12 +931,12 @@ static unsigned ComputeRefLen(size_t const count, uint32_t const cigar[])
{
unsigned rslt = 0;
unsigned i;
-
+
for (i = 0; i < count; ++i) {
uint32_t const op = cigar[i];
unsigned const len = op >> 4;
int const code = op & 0x0F;
-
+
switch (code) {
case 0: /* M */
case 2: /* D */
@@ -954,7 +953,7 @@ static unsigned ComputeRefLen(size_t const count, uint32_t const cigar[])
static BAM_Record *BAM_GetRecordSliced(NGS_Refcount *const object, ctx_t ctx)
{
BAM_RefIndexSlice *const self = (BAM_RefIndexSlice *)object;
-
+
if (self->chunk == NULL || self->chunks == self->cur)
return NULL;
for ( ; ; ) {
@@ -965,7 +964,7 @@ static BAM_Record *BAM_GetRecordSliced(NGS_Refcount *const object, ctx_t ctx)
if (rec->POS > 0) {
if (rec->REFID != self->refID)
break;
-
+
uint64_t const pos = rec->POS - 1;
if (pos >= self->end)
break;
@@ -976,7 +975,7 @@ static BAM_Record *BAM_GetRecordSliced(NGS_Refcount *const object, ctx_t ctx)
if (self->cur + 1 >= self->chunks)
break;
-
+
Seek(self->parent, ctx, self->chunk[++self->cur]);
}
done = false;
@@ -994,7 +993,7 @@ static unsigned CountWhereLess(uint64_t const max,
if (max) {
unsigned count = 0;
unsigned i;
-
+
for (i = 0; i < N; ++i) {
if (chunk[i].first < max)
++count;
@@ -1011,10 +1010,10 @@ static unsigned CopyWhereLess(uint64_t rslt[], uint64_t const max,
if (max) {
unsigned count = 0;
unsigned i;
-
+
for (i = 0; i < N; ++i) {
uint64_t const first = chunk[i].first;
-
+
if (first < max) {
rslt[count] = first;
++count;
@@ -1024,10 +1023,10 @@ static unsigned CopyWhereLess(uint64_t rslt[], uint64_t const max,
}
else {
unsigned i;
-
+
for (i = 0; i < N; ++i)
rslt[i] = chunk[i].first;
-
+
return N;
}
}
@@ -1059,10 +1058,10 @@ static int IndexSlice(RefIndex const *const self,
unsigned i;
unsigned count = CountWhereLess(maxpos, self->bins[0].count,
&self->chunk[self->bins[0].offset]);
-
+
for (i = 0; i < 5; ++i) {
unsigned const shift = 14 + 3 * (4 - i);
-
+
int_beg[i] = (beg >> shift) + first[i];
int_cnt[i] = (cnt >> shift) + 1;
}
@@ -1070,10 +1069,10 @@ static int IndexSlice(RefIndex const *const self,
unsigned const beg = int_beg[i];
unsigned const N = int_cnt[i];
unsigned j;
-
+
for (j = 0; j < N; ++j) {
RefIndexBinInfo const bin = self->bins[beg + j];
-
+
count += CountWhereLess(maxpos, bin.count, &self->chunk[bin.offset]);
}
}
@@ -1088,13 +1087,13 @@ static int IndexSlice(RefIndex const *const self,
unsigned const beg = int_beg[i];
unsigned const N = int_cnt[i];
unsigned ii;
-
+
for (ii = 0; ii < N; ++ii) {
RefIndexBinInfo const bin = self->bins[beg + ii];
unsigned const copied = CopyWhereLess(&array[j], maxpos,
bin.count,
&self->chunk[bin.offset]);
-
+
j += copied;
}
}
@@ -1114,17 +1113,17 @@ static void LoadIndex_Bins(RefIndex *const self,
unsigned i;
unsigned j = 0;
size_t offset = 0;
-
+
for (i = 0; i < N; ++i) {
uint32_t const bin = LE2UInt32(data + offset + 0);
int32_t const n_chunk = LE2Int32(data + offset + 4);
-
+
if (bin == MAX_BIN && n_chunk == 2) {
uint64_t const off_beg = LE2UInt64(data + offset + 8);
uint64_t const off_end = LE2UInt64(data + offset + 16);
uint64_t const n_mapped = LE2UInt64(data + offset + 24);
uint64_t const n_unmapped = LE2UInt64(data + offset + 32);
-
+
self->off_beg = off_beg;
self->off_end = off_end;
self->n_mapped = n_mapped;
@@ -1132,13 +1131,13 @@ static void LoadIndex_Bins(RefIndex *const self,
}
else if (bin < MAX_BIN) {
unsigned ii;
-
+
self->bins[bin].count = n_chunk;
self->bins[bin].offset = j;
for (ii = 0; ii < n_chunk; ++ii) {
uint64_t const beg = LE2UInt64(data + offset + 16 * ii + 8);
/* uint64_t const end = LE2UInt64(data + offset + 16 * ii + 16); */
-
+
self->chunk[j + ii].first = beg;
/* self->chunk[j + ii].last = end; */
}
@@ -1154,10 +1153,10 @@ static void LoadIndex_Intervals(RefIndex *const self,
{
uint64_t last = 0;
unsigned i;
-
+
for (i = 0; i < N; ++i) {
uint64_t const intvl = LE2UInt64(data + 8 * i);
-
+
self->interval[i] = intvl == last ? 0 : intvl;
last = intvl;
}
@@ -1174,17 +1173,17 @@ static size_t LoadIndex_3(HeaderRefInfo *const self, char const data[],
unsigned chunks = 0;
size_t offset = 4;
unsigned i;
-
+
if (n_bin < 0)
return 0;
-
+
for (i = 0; i < n_bin; ++i) {
if ((void const *)(data + offset + 8) < endp) {
int32_t const n_chunk = LE2Int32(data + offset + 4);
-
+
if (n_chunk < 0)
return 0;
-
+
chunks += n_chunk;
offset += 8 + 16 * n_chunk;
continue;
@@ -1193,7 +1192,7 @@ static size_t LoadIndex_3(HeaderRefInfo *const self, char const data[],
}
if ((void const *)(data + offset + 4) < endp) {
int32_t const n_intv = LE2Int32(data + offset);
-
+
if ((void const *)(data + offset + 4 + n_intv * 8) <= endp) {
self->index = calloc(1, ((uint8_t const *)&((RefIndex const *)NULL)->chunk[chunks])-((uint8_t const *)NULL));
if (self->index) {
@@ -1212,30 +1211,30 @@ static void LoadIndex_2(BAM_ReadCollection *const self, ctx_t ctx,
size_t const datasize, char const data[])
{
void const *const endp = data + datasize;
-
+
if (datasize >= 8 && memcmp(data, "BAI\1", 4) == 0) {
int32_t const n_ref = LE2Int32(data + 4);
-
+
if (n_ref == self->references->count) {
size_t offset = 8;
unsigned i;
-
+
for (i = 0; i < n_ref; ++i) {
size_t const size = LoadIndex_3(&self->references->ref[i], data + offset, endp);
-
+
if (size == 0)
goto BAD;
-
+
offset += size;
}
}
}
return;
-
+
BAD:
{
unsigned i;
-
+
for (i = 0; i < self->references->count; ++i) {
free(self->references->ref[i].index);
self->references->ref[i].index = NULL;
@@ -1247,7 +1246,7 @@ static rc_t OpenIndex(KFile const **const rslt, char const basename[])
{
KDirectory *dir;
rc_t rc = KDirectoryNativeDir(&dir);
-
+
if (rc == 0) {
rc = KDirectoryOpenFileRead(dir, rslt, "%s.bai", basename);
KDirectoryRelease(dir);
@@ -1259,12 +1258,12 @@ static void LoadIndex(BAM_ReadCollection *const self, ctx_t ctx)
{
KFile const *fp;
rc_t rc = OpenIndex(&fp, self->path);
-
+
if (rc == 0) {
uint64_t fsize;
size_t nread;
char *data;
-
+
rc = KFileSize(fp, &fsize);
data = malloc(fsize);
if (data) {
@@ -1272,10 +1271,16 @@ static void LoadIndex(BAM_ReadCollection *const self, ctx_t ctx)
if (rc == 0 && nread == fsize) {
LoadIndex_2(self, ctx, fsize, data);
}
+ else {
+ INTERNAL_ERROR ( xcUnexpected, "KFileReadAll(%lu) rc = %R nread = %lu", fsize, rc, (uint64_t)nread );
+ }
free(data);
}
KFileRelease(fp);
}
+ else {
+ INTERNAL_ERROR ( xcUnexpected, "OpenIndex(%s) rc = %R", self->path, rc );
+ }
}
static void BAM_ReferenceInfoWhack(BAM_ReferenceInfo *const self)
@@ -1294,7 +1299,7 @@ static void LoadHeaderRefs(BAM_ReadCollection *const self, ctx_t ctx)
unsigned i;
unsigned const nrefs = self->references->count;
HeaderRefInfo *const ref = &self->references->ref[0];
-
+
for (i = 0; i < nrefs; ++i) {
TRY(uint32_t const namelen = ReadU32(self, ctx)) {
if (namelen > INT32_MAX) {
@@ -1359,7 +1364,7 @@ static bool HeaderCheckSignature(BAM_ReadCollection *const self, ctx_t ctx)
FUNC_ENTRY(ctx, rcSRA, rcFile, rcOpening);
static char const sig[4] = "BAM\1";
char act[4];
-
+
if (ReadN(self, ctx, 4, act) == 4)
return memcmp(sig, act, 4) == 0;
return false;
@@ -1398,7 +1403,7 @@ static KFile const *OpenRead(ctx_t ctx, char const path[])
TRY(KDirectory *const dir = GetCWD(ctx))
{
rc_t const rc = KDirectoryOpenFileRead(dir, &fp, path);
-
+
KDirectoryRelease(dir);
if (rc) {
USER_ERROR(xcUnexpected, "'%s' failed to open for read rc = %R", path, rc);
@@ -1425,11 +1430,11 @@ static char *DuplicatePath(ctx_t ctx, char const path[], size_t const n)
static size_t LastPathElement(size_t const length, char const path[/* length */])
{
size_t i = length;
-
+
while (i) {
size_t const j = i - 1;
int const ch = path[j];
-
+
if (ch == '/')
return i;
@@ -1441,7 +1446,7 @@ static size_t LastPathElement(size_t const length, char const path[/* length */]
static void InflateInit(BAM_ReadCollection *const self, ctx_t ctx)
{
int const zrc = inflateInit2(&self->zs, MAX_WBITS + 16);
-
+
switch (zrc) {
case Z_OK:
break;
@@ -1459,14 +1464,14 @@ BAM_ReadCollection *BAM_ReadCollectionInit(BAM_ReadCollection *const self, ctx_t
{
FUNC_ENTRY(ctx, rcSRA, rcFile, rcOpening);
size_t const namelen = strlen(path);
-
+
TRY(char *const pathcopy = DuplicatePath(ctx, path, namelen)) {
size_t const namestart = LastPathElement(namelen, pathcopy);
-
+
self->path = pathcopy;
self->namelen = (unsigned)(namelen - namestart);
self->namestart = (unsigned)namestart;
-
+
TRY(self->fp = OpenRead(ctx, path)) {
TRY(self->iobuffer = AllocIOBuffer(ctx)) {
TRY(InflateInit(self, ctx)) {
@@ -1489,7 +1494,7 @@ NGS_ReadCollection * NGS_ReadCollectionMakeBAM(ctx_t ctx, char const path[])
{
/* NGS_Refcount */
{ BAM_ReadCollectionWhack },
-
+
/* NGS_ReadCollection */
BAM_ReadCollectionName,
BAM_ReadCollectionReadGroups,
diff --git a/libs/ngs/CSRA1_Alignment.c b/libs/ngs/CSRA1_Alignment.c
index 98cdcba..952e095 100644
--- a/libs/ngs/CSRA1_Alignment.c
+++ b/libs/ngs/CSRA1_Alignment.c
@@ -920,7 +920,7 @@ bool CSRA1_AlignmentIteratorNext ( CSRA1_Alignment* self, ctx_t ctx )
self -> cur_row = self -> secondary_start;
self -> row_max = self -> secondary_max;
- // let's re-run "next" again to check SEQ_SPOT_ID
+ /* let's re-run "next" again to check SEQ_SPOT_ID */
self -> seen_first = false;
return CSRA1_AlignmentIteratorNext ( self, ctx );
}
diff --git a/libs/ngs/CSRA1_Read.c b/libs/ngs/CSRA1_Read.c
index 4290962..cd23259 100644
--- a/libs/ngs/CSRA1_Read.c
+++ b/libs/ngs/CSRA1_Read.c
@@ -51,6 +51,8 @@
# define min(a,b) ( (a) < (b) ? (a) : (b) )
#endif
+static bool CSRA1_ReadIteratorNext ( CSRA1_Read * self, ctx_t ctx );
+
/*--------------------------------------------------------------------------
* CSRA1_Read
*/
@@ -88,7 +90,7 @@ static NGS_Read_vt CSRA1_Read_vt_inst =
SRA_ReadGetQualities,
SRA_ReadNumFragments,
CSRA1_ReadFragIsAligned,
- SRA_ReadIteratorNext,
+ CSRA1_ReadIteratorNext,
SRA_ReadIteratorGetCount,
};
@@ -461,3 +463,100 @@ bool CSRA1_ReadFragIsAligned ( CSRA1_Read * cself, ctx_t ctx, uint32_t frag_idx
CLEAR();
return false;
}
+
+bool
+CSRA1_ReadIteratorNext ( CSRA1_Read * cself, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcAccessing );
+ SRA_Read * self;
+
+ assert ( cself != NULL );
+
+ self = & cself -> dad;
+
+ if ( self -> wants_full )
+ {
+ return SRA_ReadIteratorNext ( cself, ctx );
+ }
+
+ /* to iterate over partially aligned and unaligned reads, use column CMP_READ */
+
+ self -> seen_first_frag = false;
+ self -> seen_last_frag = false;
+
+ self -> cur_frag = 0;
+ self -> bio_frags = 0;
+ self -> frag_idx = 0;
+ self -> frag_max = 0;
+ self -> frag_start = 0;
+ self -> frag_len = 0;
+
+ self -> READ_TYPE = NULL;
+ self -> READ_LEN = NULL;
+
+ if ( self -> seen_first )
+ { /* move to next row */
+ ++ self -> cur_row;
+ }
+ else
+ {
+ self -> seen_first = true;
+ }
+
+ while ( self -> cur_row < self -> row_max )
+ {
+ enum NGS_ReadCategory cat;
+ NGS_String * read;
+ ON_FAIL ( read = NGS_CursorGetString ( self -> curs, ctx, self -> cur_row, seq_CMP_READ ) )
+ return false;
+
+ if ( NGS_StringSize ( read, ctx ) == 0 )
+ { // aligned read - skip
+ ++ self -> cur_row;
+ continue;
+ }
+
+ /* work the category filter, we know wants_full is false */
+ ON_FAIL ( cat = SRA_ReadGetCategory ( cself, ctx ) )
+ return false;
+
+ if ( ( cat == NGS_ReadCategory_partiallyAligned && ! self -> wants_partial )
+ ||
+ ( cat == NGS_ReadCategory_unaligned && ! self -> wants_unaligned ) )
+ {
+ ++ self -> cur_row;
+ continue;
+ }
+
+ /* work the read group filter if required */
+ if ( self -> group_name != NULL )
+ {
+ uint32_t size;
+
+ ON_FAIL ( NGS_String* group = NGS_CursorGetString ( self -> curs, ctx, self -> cur_row, seq_GROUP ) )
+ return false;
+
+ size = ( uint32_t ) NGS_StringSize ( group, ctx );
+ if ( string_cmp ( NGS_StringData ( self -> group_name, ctx ),
+ NGS_StringSize ( self -> group_name, ctx ),
+ NGS_StringData ( group, ctx ),
+ size,
+ size ) != 0 )
+ {
+ NGS_StringRelease ( group, ctx );
+ ++ self -> cur_row;
+ continue;
+ }
+ NGS_StringRelease ( group, ctx );
+ }
+
+ TRY ( SRA_ReadIteratorInitFragment ( cself, ctx ) )
+ {
+ return true;
+ }
+
+ break;
+ }
+
+ return false;
+}
diff --git a/libs/ngs/CSRA1_ReadCollection.c b/libs/ngs/CSRA1_ReadCollection.c
index a497fe1..1edf761 100644
--- a/libs/ngs/CSRA1_ReadCollection.c
+++ b/libs/ngs/CSRA1_ReadCollection.c
@@ -130,7 +130,7 @@ void CSRA1_ReadCollectionWhack ( CSRA1_ReadCollection * self, ctx_t ctx )
VDatabaseRelease ( self -> db );
}
-static const char * reference_col_specs [] =
+const char * reference_col_specs [] =
{
"(bool)CIRCULAR",
"(utf8)NAME",
@@ -331,7 +331,7 @@ NGS_Reference * CSRA1_ReadCollectionGetReference ( CSRA1_ReadCollection * self,
ret = CSRA1_ReferenceMake ( ctx, & self -> dad, self -> db, curs, spec, self -> primaryId_count );
#if ! 0
- // release cursor if we generate new cursor for the reference each time
+ /* release cursor if we generate new cursor for the reference each time */
NGS_CursorRelease ( curs, ctx );
#endif
diff --git a/libs/ngs/CSRA1_Reference.c b/libs/ngs/CSRA1_Reference.c
index 9442e6e..a68127d 100644
--- a/libs/ngs/CSRA1_Reference.c
+++ b/libs/ngs/CSRA1_Reference.c
@@ -32,6 +32,7 @@ typedef struct CSRA1_Reference CSRA1_Reference;
#include "NGS_ReadCollection.h"
#include "NGS_Alignment.h"
+#include "NGS_ReferenceBlobIterator.h"
#include "NGS_String.h"
#include "NGS_Cursor.h"
@@ -85,6 +86,7 @@ static struct NGS_Alignment* CSRA1_ReferenceGetAlignmentSlice ( CSRA1_Referen
static struct NGS_Pileup* CSRA1_ReferenceGetPileups ( CSRA1_Reference * self, ctx_t ctx, bool wants_primary, bool wants_secondary, uint32_t filters, int32_t map_qual );
static struct NGS_Pileup* CSRA1_ReferenceGetPileupSlice ( CSRA1_Reference * self, ctx_t ctx, uint64_t offset, uint64_t size, bool wants_primary, bool wants_secondary, uint32_t filters, int32_t map_qual );
struct NGS_Statistics* CSRA1_ReferenceGetStatistics ( const CSRA1_Reference * self, ctx_t ctx );
+static struct NGS_ReferenceBlobIterator* CSRA1_ReferenceGetBlobs ( const CSRA1_Reference * self, ctx_t ctx, uint64_t offset, uint64_t size );
static bool CSRA1_ReferenceIteratorNext ( CSRA1_Reference * self, ctx_t ctx );
static NGS_Reference_vt CSRA1_Reference_vt_inst =
@@ -106,6 +108,7 @@ static NGS_Reference_vt CSRA1_Reference_vt_inst =
CSRA1_ReferenceGetPileups,
CSRA1_ReferenceGetPileupSlice,
CSRA1_ReferenceGetStatistics,
+ CSRA1_ReferenceGetBlobs,
/* NGS_ReferenceIterator */
CSRA1_ReferenceIteratorNext,
@@ -380,7 +383,7 @@ struct NGS_String * CSRA1_ReferenceGetChunk ( CSRA1_Reference * self, ctx_t ctx,
rc_t rc;
const void* data;
uint64_t cont_size;
- TRY ( VByteBlob_ContiguousChunk ( blob, ctx, rowId, &data, &cont_size, true ) ) /* stop at a repeated row */
+ TRY ( VByteBlob_ContiguousChunk ( blob, ctx, rowId, 0, &data, &cont_size, true ) ) /* stop at a repeated row */
{
uint64_t offsetInBlob = offset % self -> chunk_size;
if ( size == (uint64_t)-1 || offsetInBlob + size > cont_size )
@@ -786,6 +789,31 @@ struct NGS_Statistics* CSRA1_ReferenceGetStatistics ( const CSRA1_Reference * se
return SRA_StatisticsMake ( ctx );
}
+struct NGS_ReferenceBlobIterator* CSRA1_ReferenceGetBlobs ( const CSRA1_Reference * self, ctx_t ctx, uint64_t offset, uint64_t size )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+
+ assert ( self );
+ if ( self -> curs == NULL )
+ {
+ USER_ERROR ( xcCursorExhausted, "No more rows available" );
+ return NULL;
+ }
+ if ( ! self -> seen_first )
+ {
+ USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
+ return NULL;
+ }
+ else
+ {
+ uint64_t startRow = self -> first_row + offset / self -> chunk_size;
+ uint64_t lastRow = size == (uint64_t)-1 ?
+ self -> last_row :
+ self -> first_row + ( offset + size - 1 ) / self -> chunk_size;
+ return NGS_ReferenceBlobIteratorMake ( ctx, self -> curs, self -> first_row, startRow, lastRow );
+ }
+}
+
bool CSRA1_ReferenceFind ( NGS_Cursor const * curs, ctx_t ctx, const char * spec, int64_t* firstRow, uint64_t* rowCount )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
@@ -818,6 +846,7 @@ bool CSRA1_ReferenceFind ( NGS_Cursor const * curs, ctx_t ctx, const char * spec
}
}
/* index not available - do a table scan */
+ if ( ! FAILED () )
{
int64_t cur_row;
int64_t end_row;
@@ -1034,6 +1063,8 @@ NGS_Reference * CSRA1_ReferenceIteratorMake ( ctx_t ctx,
*/
bool CSRA1_ReferenceIteratorNext ( CSRA1_Reference * self, ctx_t ctx )
{
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+
assert ( self != NULL );
if ( self -> curs == NULL || self -> first_row > self -> iteration_row_last)
diff --git a/libs/ngs/CSRA1_Reference.h b/libs/ngs/CSRA1_Reference.h
index 2989947..b542f9a 100644
--- a/libs/ngs/CSRA1_Reference.h
+++ b/libs/ngs/CSRA1_Reference.h
@@ -61,15 +61,16 @@ enum ReferenceTableColumns
reference_NUM_COLS
};
+extern const char * reference_col_specs [];
-struct NGS_Reference * CSRA1_ReferenceMake ( ctx_t ctx,
+struct NGS_Reference * CSRA1_ReferenceMake ( ctx_t ctx,
struct NGS_ReadCollection * coll,
const struct VDatabase * db,
- const struct NGS_Cursor * curs,
+ const struct NGS_Cursor * curs,
const char * spec,
uint64_t align_id_offset );
-struct NGS_Reference * CSRA1_ReferenceIteratorMake ( ctx_t ctx,
+struct NGS_Reference * CSRA1_ReferenceIteratorMake ( ctx_t ctx,
struct NGS_ReadCollection * coll,
const struct VDatabase * db,
const struct NGS_Cursor * curs,
diff --git a/libs/ngs/CSRA1_ReferenceWindow.c b/libs/ngs/CSRA1_ReferenceWindow.c
index b3453ff..3ebb11c 100644
--- a/libs/ngs/CSRA1_ReferenceWindow.c
+++ b/libs/ngs/CSRA1_ReferenceWindow.c
@@ -590,7 +590,62 @@ int64_t AlignmentSort ( const void * p_a, const void * p_b, void *data )
}
static
-void LoadAlignmentInfo ( CSRA1_ReferenceWindow* self, ctx_t ctx, size_t* idx, int64_t id, bool primary, int64_t offset, uint64_t size )
+bool
+ApplyFilters( CSRA1_ReferenceWindow* self, ctx_t ctx, NGS_Alignment* p_al, uint32_t* p_map_qual )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+
+ bool have_map_qual = false;
+ uint32_t map_qual;
+
+ /* test for additional filtering */
+ if ( ( self -> filters & NGS_AlignmentFilterBits_prop_mask ) != 0 )
+ {
+ TRY ( INSDC_read_filter read_filter = NGS_AlignmentGetReadFilter ( p_al, ctx ) )
+ {
+ switch ( read_filter )
+ {
+ case READ_FILTER_PASS:
+ if ( CSRA1_ReferenceWindowFilterMapQual ( self ) )
+ {
+ TRY ( map_qual = NGS_AlignmentGetMappingQuality ( p_al, ctx ) )
+ {
+ have_map_qual = true;
+ if ( CSRA1_ReferenceWindowFilterMinMapQual ( self ) )
+ {
+ /* map_qual must be >= filter level */
+ if ( map_qual < self -> map_qual )
+ return false;
+ }
+ else
+ {
+ /* map qual must be <= filter level */
+ if ( map_qual > self -> map_qual )
+ return false;
+ }
+ }
+ }
+ break;
+ case READ_FILTER_REJECT:
+ if ( CSRA1_ReferenceWindowFilterDropBad ( self ) )
+ return false;
+ break;
+ case READ_FILTER_CRITERIA:
+ if ( CSRA1_ReferenceWindowFilterDropDups ( self ) )
+ return false;
+ break;
+ case READ_FILTER_REDACTED:
+ return false;
+ break;
+ }
+ }
+ }
+ *p_map_qual = have_map_qual ? map_qual : NGS_AlignmentGetMappingQuality ( p_al, ctx );
+ return true;
+}
+
+static
+void LoadAlignmentInfo ( CSRA1_ReferenceWindow* self, ctx_t ctx, size_t* idx, int64_t id, bool primary, int64_t offset, uint64_t size, bool wraparoundOnly )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
@@ -616,78 +671,55 @@ void LoadAlignmentInfo ( CSRA1_ReferenceWindow* self, ctx_t ctx, size_t* idx, in
end_slice = self -> ref_length;
}
if ( ! CSRA1_ReferenceWindowFilterStartWithinWindow ( self ) &&
- ! CSRA1_ReferenceWindowFilterNoWraparound ( self ) &&
pos + len >= (int64_t) self -> ref_length )
- { /* account for possible carryover on a circular reference */
- pos -= self -> ref_length;
+ { /* account for possible wraparounds on a circular reference */
+ if ( wraparoundOnly )
+ {
+ if ( end_slice == self -> ref_length )
+ { /* both slice and alignment wrap around */
+ overlaps = true;
+ }
+ else
+ { /* alignment wraps around and overlaps with the slice */
+ overlaps = pos + len > self -> ref_length + offset;
+ }
+/*printf("LoadAlignmentInfo(offset=%li, size=%lu) pos=%li len=%li overlaps=%i\n", offset, size, pos, len, (int)overlaps);*/
+ }
+ else
+ { /* ignore the wraparound */
+ overlaps = false;
+ }
+ }
+ else if ( wraparoundOnly )
+ {
+ overlaps = false;
+ }
+ else
+ {
+ overlaps = pos < end_slice && ( pos + len > offset );
}
- overlaps = pos < end_slice && ( pos + len > offset );
}
-
- /* use single-pass loop as a sort of sanctimonious goto mechanism */
- while ( overlaps )
+ else if ( ! CSRA1_ReferenceWindowFilterStartWithinWindow ( self ) && wraparoundOnly )
{
- int32_t map_qual = 0;
- bool have_map_qual = false;
-
- /* test for additional filtering */
- if ( ( self -> filters & NGS_AlignmentFilterBits_prop_mask ) != 0 )
+ if ( pos + len < (int64_t) self -> ref_length )
{
- TRY ( INSDC_read_filter read_filter = NGS_AlignmentGetReadFilter ( al, ctx ) )
- {
- switch ( read_filter )
- {
- case READ_FILTER_PASS:
- if ( CSRA1_ReferenceWindowFilterMapQual ( self ) )
- {
- TRY ( map_qual = NGS_AlignmentGetMappingQuality ( al, ctx ) )
- {
- have_map_qual = true;
- if ( CSRA1_ReferenceWindowFilterMinMapQual ( self ) )
- {
- /* map_qual must be >= filter level */
- if ( map_qual < self -> map_qual )
- overlaps = false;
- }
- else
- {
- /* map qual must be <= filter level */
- if ( map_qual > self -> map_qual )
- overlaps = false;
- }
- }
- }
- break;
- case READ_FILTER_REJECT:
- if ( CSRA1_ReferenceWindowFilterDropBad ( self ) )
- overlaps = false;
- break;
- case READ_FILTER_CRITERIA:
- if ( CSRA1_ReferenceWindowFilterDropDups ( self ) )
- overlaps = false;
- break;
- case READ_FILTER_REDACTED:
- overlaps = false;
- break;
- }
-
- if ( ! overlaps )
- break;
- }
+ overlaps = false;
}
+ }
- /* accept record */
-
- /*printf("%li, %li, %i, %li\n", pos, len, NGS_AlignmentGetMappingQuality ( al, ctx ), id); */
- self -> align_info [ *idx ] . id = id;
- self -> align_info [ *idx ] . pos = pos;
- self -> align_info [ *idx ] . len = len;
- self -> align_info [ *idx ] . cat = primary ? Primary : Secondary;
- self -> align_info [ *idx ] . mapq = have_map_qual ? map_qual : NGS_AlignmentGetMappingQuality ( al, ctx );
- ++ ( * idx );
-
- /* MUST break here to exit single pass */
- break;
+ if ( overlaps )
+ {
+ uint32_t map_qual;
+ if ( ApplyFilters ( self, ctx, al, &map_qual ) )
+ { /* accept record */
+/*printf("pos=%li, len=%li, end=%li, q=%i, id=%li, wrap=%i\n", pos, len, pos+len, NGS_AlignmentGetMappingQuality ( al, ctx ), id, wraparoundOnly);*/
+ self -> align_info [ *idx ] . id = id;
+ self -> align_info [ *idx ] . pos = pos;
+ self -> align_info [ *idx ] . len = len;
+ self -> align_info [ *idx ] . cat = primary ? Primary : Secondary;
+ self -> align_info [ *idx ] . mapq = map_qual;
+ ++ ( * idx );
+ }
}
}
@@ -759,7 +791,7 @@ int64_t AlignmentSortCircular ( const void * p_a, const void * p_b, void *data )
}
static
-void LoadAlignments ( CSRA1_ReferenceWindow* self, ctx_t ctx, int64_t chunk_row_id, int64_t offset, uint64_t size )
+void LoadAlignments ( CSRA1_ReferenceWindow* self, ctx_t ctx, int64_t chunk_row_id, int64_t offset, uint64_t size, bool wraparounds )
{ /* append alignments for the specified chunk to self -> align_info */
const int64_t* primary_idx = NULL;
uint32_t primary_idx_end = 0;
@@ -803,12 +835,12 @@ void LoadAlignments ( CSRA1_ReferenceWindow* self, ctx_t ctx, int64_t chunk_row_
uint32_t i;
for ( i = 0; i < primary_idx_end; ++i )
{
- ON_FAIL ( LoadAlignmentInfo( self, ctx, & self -> align_info_total, primary_idx [ i ], true, offset, size ) )
+ ON_FAIL ( LoadAlignmentInfo( self, ctx, & self -> align_info_total, primary_idx [ i ], true, offset, size, wraparounds ) )
return;
}
for ( i = 0; i < secondary_idx_end; ++i )
{
- ON_FAIL ( LoadAlignmentInfo( self, ctx, & self -> align_info_total, secondary_idx [ i ] + self -> id_offset, false, offset, size ) )
+ ON_FAIL ( LoadAlignmentInfo( self, ctx, & self -> align_info_total, secondary_idx [ i ] + self -> id_offset, false, offset, size, wraparounds ) )
return;
}
}
@@ -831,18 +863,23 @@ bool LoadFirstCircular ( CSRA1_ReferenceWindow* self, ctx_t ctx )
{ /* load the last chunk of the reference, to cover possible overlaps into the first chunk */
if ( self -> slice_size == 0 )
{ /* loading possible overlaps with the first chunk */
- ON_FAIL ( LoadAlignments ( self, ctx, last_chunk, 0, self -> chunk_size ) )
+ ON_FAIL ( LoadAlignments ( self, ctx, last_chunk, 0, self -> chunk_size, true ) )
return false;
}
else if ( self -> slice_offset < self -> chunk_size )
- { /* loading possible overlaps with a slice inside the first chunk */
- ON_FAIL ( LoadAlignments ( self, ctx, last_chunk, self -> slice_offset, self -> chunk_size - self -> slice_offset ) )
+ { /* the slice starts in the first chunk; load alignments wrapped around from the last chunk */
+ ON_FAIL ( LoadAlignments ( self, ctx, last_chunk, self -> slice_offset, self -> chunk_size - self -> slice_offset, true ) )
+ return false;
+ }
+ else if ( self -> slice_offset + self -> slice_size > self -> ref_length )
+ { /* the slice starts in the last and wraps around to the first chunk; load alignments wrapped around into the first chunk */
+ ON_FAIL ( LoadAlignments ( self, ctx, last_chunk, self -> slice_offset, self -> slice_size, true ) )
return false;
}
/* target slice is not in the first chunk, no need to look for overlaps from the end of the reference */
}
- ON_FAIL ( LoadAlignments ( self, ctx, self -> ref_begin, self -> slice_offset, self -> slice_size ) )
+ ON_FAIL ( LoadAlignments ( self, ctx, self -> ref_begin, self -> slice_offset, self -> slice_size, false ) )
return false;
if ( self -> align_info_total > 0 )
@@ -864,7 +901,7 @@ bool LoadNextChunk ( CSRA1_ReferenceWindow* self, ctx_t ctx )
self -> align_info_total = 0;
while ( self -> ref_begin < self -> ref_end )
{
- ON_FAIL ( LoadAlignments ( self, ctx, self -> ref_begin, self -> slice_offset, self -> slice_size ) )
+ ON_FAIL ( LoadAlignments ( self, ctx, self -> ref_begin, self -> slice_offset, self -> slice_size, false ) )
return false;
if ( self -> align_info_total > 0 )
diff --git a/libs/ngs/Makefile b/libs/ngs/Makefile
index 9fa6100..e52c9d4 100644
--- a/libs/ngs/Makefile
+++ b/libs/ngs/Makefile
@@ -113,8 +113,10 @@ NGS_SRC = \
NGS_Id \
NGS_ErrBlock \
NGS_FragmentBlob \
- NGS_FragmentBlobIterator \
- VByteBlob \
+ NGS_FragmentBlobIterator \
+ NGS_ReferenceBlob \
+ NGS_ReferenceBlobIterator \
+ VByteBlob \
NGS_OBJ = \
$(addsuffix .$(LOBX),$(NGS_SRC))
diff --git a/libs/ngs/NGS_FragmentBlob.c b/libs/ngs/NGS_FragmentBlob.c
index 43ec0e7..05c1557 100644
--- a/libs/ngs/NGS_FragmentBlob.c
+++ b/libs/ngs/NGS_FragmentBlob.c
@@ -54,7 +54,6 @@ struct NGS_FragmentBlob
uint64_t size; /* from the start of the first row until the end of the blob */
const NGS_String* run;
- const NGS_Cursor* curs;
const VBlob* blob_READ;
const VBlob* blob_READ_LEN;
const VBlob* blob_READ_TYPE;
@@ -70,7 +69,6 @@ NGS_FragmentBlobWhack ( NGS_FragmentBlob * self, ctx_t ctx )
VBlobRelease ( (VBlob*) self -> blob_READ );
VBlobRelease ( (VBlob*) self -> blob_READ_LEN );
VBlobRelease ( (VBlob*) self -> blob_READ_TYPE );
- NGS_CursorRelease ( self -> curs, ctx );
NGS_StringRelease ( self -> run, ctx );
}
}
@@ -113,8 +111,7 @@ NGS_FragmentBlobMake ( ctx_t ctx, const NGS_String* run, const struct NGS_Cursor
TRY ( ret -> blob_READ_TYPE = NGS_CursorGetVBlob ( curs, ctx, rowId, seq_READ_TYPE ) );
{
ret -> rowId = rowId;
- ret -> curs = NGS_CursorDuplicate ( curs, ctx );
- TRY ( VByteBlob_ContiguousChunk ( ret -> blob_READ, ctx, ret -> rowId, &ret -> data, &ret -> size, false ) )
+ TRY ( VByteBlob_ContiguousChunk ( ret -> blob_READ, ctx, ret -> rowId, 0, &ret -> data, &ret -> size, false ) )
{
return ret;
}
@@ -215,6 +212,23 @@ NGS_FragmentBlobSize ( const struct NGS_FragmentBlob * self, ctx_t ctx )
return 0;
}
+const struct NGS_String *
+NGS_FragmentBlobRun ( const struct NGS_FragmentBlob * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
+
+ if ( self == NULL )
+ {
+ INTERNAL_ERROR ( xcParamNull, "bad object reference" );
+ }
+ else
+ {
+ return self -> run;
+ }
+ return 0;
+}
+
+
static
void
GetFragInfo ( const NGS_FragmentBlob * self, ctx_t ctx, int64_t p_rowId, uint64_t p_offsetInRow, uint64_t* fragStart, uint64_t* baseCount, int32_t* bioNumber )
@@ -364,9 +378,6 @@ NGS_FragmentBlobInfoByOffset ( const struct NGS_FragmentBlob * self, ctx_t ctx,
elem_count_t length = PageMapIteratorDataLength ( &pmIt );
elem_count_t offset = PageMapIteratorDataOffset ( &pmIt );
row_count_t repeat = PageMapIteratorRepeatCount ( &pmIt );
-/*TODO: with the updated design:
- assert ( repeat == 1 );
- */
if ( offsetInBases < offset + length * repeat )
{
while ( repeat > 1 )
@@ -377,6 +388,7 @@ NGS_FragmentBlobInfoByOffset ( const struct NGS_FragmentBlob * self, ctx_t ctx,
}
offset += length;
++rowInBlob;
+ --repeat;
}
if ( rowId != NULL )
{
diff --git a/libs/ngs/NGS_FragmentBlob.h b/libs/ngs/NGS_FragmentBlob.h
index 613ab16..abdb38f 100644
--- a/libs/ngs/NGS_FragmentBlob.h
+++ b/libs/ngs/NGS_FragmentBlob.h
@@ -77,6 +77,11 @@ const void* NGS_FragmentBlobData ( const struct NGS_FragmentBlob * self, ctx_t c
*/
uint64_t NGS_FragmentBlobSize ( const struct NGS_FragmentBlob * self, ctx_t ctx );
+/* Run
+ * Returns the name of the run containing the blob
+ */
+const struct NGS_String * NGS_FragmentBlobRun ( const struct NGS_FragmentBlob * self, ctx_t ctx );
+
/* InfoByOffset
* retrieve fragment info by offset in the blob
* offsetInBases - an offset into the blob
@@ -115,6 +120,7 @@ void NGS_FragmentBlobInfoByRowId ( const struct NGS_FragmentBlob * self, ctx_t c
* fragmentNumber - 0-based biological fragment number inside the read
*
* returns the ID string of the specified fragment
+ * DEPRECATED. Call NGS_FragmentBlobInfoByRowId(..., rowId, fragNumber, ...) and NGS_IdMakeFragment ( ctx, NGS_FragmentBlobRun(...), false, rowId, fragNumber );
*/
struct NGS_String* NGS_FragmentBlobMakeFragmentId ( const struct NGS_FragmentBlob * self, ctx_t ctx, int64_t rowId, uint32_t fragNumber );
diff --git a/libs/ngs/NGS_FragmentBlobIterator.c b/libs/ngs/NGS_FragmentBlobIterator.c
index 4baa0c1..fc6a608 100644
--- a/libs/ngs/NGS_FragmentBlobIterator.c
+++ b/libs/ngs/NGS_FragmentBlobIterator.c
@@ -32,6 +32,8 @@
#include <ngs/itf/Refcount.h>
+#include <klib/rc.h>
+
#include <vdb/cursor.h>
#include "NGS_String.h"
@@ -178,6 +180,10 @@ NGS_FragmentBlobIteratorNext ( NGS_FragmentBlobIterator * self, ctx_t ctx )
NGS_FragmentBlobRelease ( ret, ctx );
}
}
+ else if ( GetRCState ( rc ) != rcNotFound )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "VCursorFindNextRowIdDirect(READ, row=%li ) rc = %R", self -> next_row, rc );
+ }
self -> next_row = self -> last_row + 1;
}
diff --git a/libs/ngs/NGS_ReadCollection.c b/libs/ngs/NGS_ReadCollection.c
index 7f70e63..2c15b0e 100644
--- a/libs/ngs/NGS_ReadCollection.c
+++ b/libs/ngs/NGS_ReadCollection.c
@@ -363,9 +363,9 @@ bool NGS_ReadCollectionHasReference ( NGS_ReadCollection * self, ctx_t ctx, cons
if ( self == NULL )
INTERNAL_WARNING ( xcSelfNull, "failed to get reference '%.128s'", spec );
else if ( spec == NULL )
- INTERNAL_WARNING ( xcParamNull, "reference spec" );
+ INTERNAL_WARNING ( xcParamNull, "NULL reference spec" );
else if ( spec [ 0 ] == 0 )
- INTERNAL_WARNING ( xcStringEmpty, "reference spec" );
+ INTERNAL_WARNING ( xcStringEmpty, "empty reference spec" );
else
{
POP_CTX ( ctx );
@@ -381,9 +381,9 @@ struct NGS_Reference * NGS_ReadCollectionGetReference ( NGS_ReadCollection * sel
if ( self == NULL )
INTERNAL_ERROR ( xcSelfNull, "failed to get reference '%.128s'", spec );
else if ( spec == NULL )
- INTERNAL_ERROR ( xcParamNull, "reference spec" );
+ INTERNAL_ERROR ( xcParamNull, "NULL reference spec" );
else if ( spec [ 0 ] == 0 )
- INTERNAL_ERROR ( xcStringEmpty, "reference spec" );
+ INTERNAL_ERROR ( xcStringEmpty, "empty reference spec" );
else
{
POP_CTX ( ctx );
diff --git a/libs/ngs/NGS_Reference.c b/libs/ngs/NGS_Reference.c
index 2fb07fe..9551dff 100644
--- a/libs/ngs/NGS_Reference.c
+++ b/libs/ngs/NGS_Reference.c
@@ -48,7 +48,7 @@
#define Self( obj ) \
( ( NGS_Reference* ) ( obj ) )
-
+
static NGS_String_v1 * ITF_Reference_v1_get_cmn_name ( const NGS_Reference_v1 * self, NGS_ErrBlock_v1 * err )
{
HYBRID_FUNC_ENTRY ( rcSRA, rcRefcount, rcAccessing );
@@ -179,17 +179,17 @@ uint32_t align_flags_to_filters ( uint32_t flags )
( ( flags ) >> 2 )
#endif
-static struct NGS_Alignment_v1 * ITF_Reference_v1_get_filtered_alignments ( const NGS_Reference_v1 * self,
- NGS_ErrBlock_v1 * err,
- enum NGS_ReferenceAlignFlags flags,
+static struct NGS_Alignment_v1 * ITF_Reference_v1_get_filtered_alignments ( const NGS_Reference_v1 * self,
+ NGS_ErrBlock_v1 * err,
+ enum NGS_ReferenceAlignFlags flags,
int32_t map_qual )
{
HYBRID_FUNC_ENTRY ( rcSRA, rcRefcount, rcAccessing );
-
+
bool wants_primary = ( flags & NGS_ReferenceAlignFlags_wants_primary ) != 0;
bool wants_secondary = ( flags & NGS_ReferenceAlignFlags_wants_secondary ) != 0;
uint32_t filters = align_flags_to_filters ( flags );
-
+
/*TODO: reject unimplemented flags */
ON_FAIL ( struct NGS_Alignment * ret = NGS_ReferenceGetFilteredAlignments ( Self ( self ), ctx,
wants_primary, wants_secondary, filters, map_qual ) )
@@ -213,19 +213,19 @@ static struct NGS_Alignment_v1 * ITF_Reference_v1_get_align_slice ( const NGS_Re
return ( struct NGS_Alignment_v1 * ) ret;
}
-static struct NGS_Alignment_v1 * ITF_Reference_v1_get_filtered_align_slice ( const NGS_Reference_v1 * self,
- NGS_ErrBlock_v1 * err,
- int64_t start,
- uint64_t length,
- enum NGS_ReferenceAlignFlags flags,
+static struct NGS_Alignment_v1 * ITF_Reference_v1_get_filtered_align_slice ( const NGS_Reference_v1 * self,
+ NGS_ErrBlock_v1 * err,
+ int64_t start,
+ uint64_t length,
+ enum NGS_ReferenceAlignFlags flags,
int32_t map_qual )
{
HYBRID_FUNC_ENTRY ( rcSRA, rcRefcount, rcAccessing );
-
+
bool wants_primary = ( flags & NGS_ReferenceAlignFlags_wants_primary ) != 0;
bool wants_secondary = ( flags & NGS_ReferenceAlignFlags_wants_secondary ) != 0;
uint32_t filters = align_flags_to_filters ( flags );
-
+
/*TODO: reject unimplemented flags */
ON_FAIL ( struct NGS_Alignment * ret = NGS_ReferenceGetFilteredAlignmentSlice ( Self ( self ), ctx, start, length, wants_primary, wants_secondary, filters, map_qual ) )
{
@@ -363,7 +363,7 @@ NGS_Reference_v1_vt ITF_Reference_vt =
/* 1.2 */
ITF_Reference_v1_get_align_count,
-
+
/* 1.3 */
ITF_Reference_v1_get_filtered_alignments,
ITF_Reference_v1_get_filtered_align_slice
@@ -375,21 +375,21 @@ NGS_Reference_v1_vt ITF_Reference_vt =
*/
#define VT( self, msg ) \
( ( ( const NGS_Reference_vt* ) ( self ) -> dad . vt ) -> msg )
-
+
/* Init
*/
-void NGS_ReferenceInit ( ctx_t ctx,
- struct NGS_Reference * self,
- NGS_Reference_vt * vt,
- const char *clsname,
- const char *instname,
+void NGS_ReferenceInit ( ctx_t ctx,
+ struct NGS_Reference * self,
+ NGS_Reference_vt * vt,
+ const char *clsname,
+ const char *instname,
struct NGS_ReadCollection * coll )
{
FUNC_ENTRY ( ctx, rcSRA, rcRow, rcConstructing );
-
+
assert ( self );
assert ( vt );
-
+
TRY ( NGS_RefcountInit ( ctx, & self -> dad, & ITF_Reference_vt . dad, & vt -> dad, clsname, instname ) )
{
assert ( vt -> get_common_name != NULL );
@@ -406,8 +406,9 @@ void NGS_ReferenceInit ( ctx_t ctx,
assert ( vt -> get_pileup_slice != NULL );
assert ( vt -> get_statistics != NULL );
assert ( vt -> next != NULL );
+ assert ( vt -> get_blobs != NULL );
}
-
+
assert ( coll );
self -> coll = NGS_ReadCollectionDuplicate ( coll, ctx );
}
@@ -416,7 +417,7 @@ void NGS_ReferenceWhack( NGS_Reference * self, ctx_t ctx )
{
NGS_ReadCollectionRelease ( self -> coll, ctx );
}
-
+
/*--------------------------------------------------------------------------
* NGS_ReferenceIterator
*/
@@ -553,7 +554,7 @@ struct NGS_Alignment* NGS_ReferenceGetAlignments ( NGS_Reference * self, ctx_t c
}
else
{
- // alignment iterator does not filter out bad reads and duplicates by default
+ /* alignment iterator does not filter out bad reads and duplicates by default */
const uint32_t filters = NGS_AlignmentFilterBits_pass_bad | NGS_AlignmentFilterBits_pass_dups;
return VT ( self, get_alignments ) ( self, ctx, wants_primary, wants_secondary, filters, 0 );
}
@@ -598,11 +599,11 @@ uint64_t NGS_ReferenceGetAlignmentCount ( NGS_Reference * self, ctx_t ctx, bool
/* GetAlignmentSlice
*/
-struct NGS_Alignment* NGS_ReferenceGetAlignmentSlice ( NGS_Reference * self,
- ctx_t ctx,
- uint64_t offset,
+struct NGS_Alignment* NGS_ReferenceGetAlignmentSlice ( NGS_Reference * self,
+ ctx_t ctx,
+ uint64_t offset,
uint64_t size,
- bool wants_primary,
+ bool wants_primary,
bool wants_secondary )
{
if ( self == NULL )
@@ -612,7 +613,7 @@ struct NGS_Alignment* NGS_ReferenceGetAlignmentSlice ( NGS_Reference * self,
}
else
{
- // alignment iterator does not filter out bad reads and duplicates by default
+ /* alignment iterator does not filter out bad reads and duplicates by default */
const uint32_t filters = NGS_AlignmentFilterBits_pass_bad | NGS_AlignmentFilterBits_pass_dups;
return VT ( self, get_slice ) ( self, ctx, offset, size, wants_primary, wants_secondary, filters, 0 );
}
@@ -622,7 +623,7 @@ struct NGS_Alignment* NGS_ReferenceGetAlignmentSlice ( NGS_Reference * self,
/* GetFilteredAlignmentSlice
*/
-struct NGS_Alignment* NGS_ReferenceGetFilteredAlignmentSlice ( NGS_Reference * self,
+struct NGS_Alignment* NGS_ReferenceGetFilteredAlignmentSlice ( NGS_Reference * self,
ctx_t ctx, uint64_t offset, uint64_t size, bool wants_primary, bool wants_secondary, uint32_t filters, int32_t map_qual )
{
if ( self == NULL )
@@ -649,7 +650,7 @@ struct NGS_Pileup* NGS_ReferenceGetPileups ( NGS_Reference * self, ctx_t ctx, bo
}
else
{
- // pileup filters out bad reads and duplicates by default
+ /* pileup filters out bad reads and duplicates by default */
return VT ( self, get_pileups ) ( self, ctx, wants_primary, wants_secondary, 0, 0 );
}
@@ -675,12 +676,12 @@ struct NGS_Pileup* NGS_ReferenceGetFilteredPileups ( NGS_Reference * self, ctx_t
/* GetPileupSlice
*/
-struct NGS_Pileup* NGS_ReferenceGetPileupSlice ( NGS_Reference * self,
- ctx_t ctx,
- uint64_t offset,
+struct NGS_Pileup* NGS_ReferenceGetPileupSlice ( NGS_Reference * self,
+ ctx_t ctx,
+ uint64_t offset,
uint64_t size,
- bool wants_primary,
- bool wants_secondary )
+ bool wants_primary,
+ bool wants_secondary )
{
if ( self == NULL )
{
@@ -697,14 +698,14 @@ struct NGS_Pileup* NGS_ReferenceGetPileupSlice ( NGS_Reference * self,
/* GetFilteredPileupSlice
*/
-struct NGS_Pileup* NGS_ReferenceGetFilteredPileupSlice ( NGS_Reference * self,
- ctx_t ctx,
- uint64_t offset,
+struct NGS_Pileup* NGS_ReferenceGetFilteredPileupSlice ( NGS_Reference * self,
+ ctx_t ctx,
+ uint64_t offset,
uint64_t size,
- bool wants_primary,
+ bool wants_primary,
bool wants_secondary,
uint32_t filters,
- int32_t map_qual )
+ int32_t map_qual )
{
if ( self == NULL )
{
@@ -736,11 +737,28 @@ struct NGS_Statistics* NGS_ReferenceGetStatistics ( const NGS_Reference * self,
return NULL;
}
+/* GetBlobs
+ */
+struct NGS_ReferenceBlobIterator* NGS_ReferenceGetBlobs ( NGS_Reference * self, ctx_t ctx, uint64_t offset, uint64_t size )
+{
+ if ( self == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
+ INTERNAL_ERROR ( xcSelfNull, "failed to get blobs" );
+ }
+ else
+ {
+ return VT ( self, get_blobs ) ( self, ctx, offset, size );
+ }
+
+ return NULL;
+}
+
/*--------------------------------------------------------------------------
* NGS_ReferenceIterator
*/
-
+
/* Next
*/
bool NGS_ReferenceIteratorNext ( NGS_Reference * self, ctx_t ctx )
@@ -830,11 +848,11 @@ static uint64_t Null_ReferenceGetAlignmentCount ( const NGS_Reference * self, ct
return 0;
}
-static struct NGS_Alignment* Null_ReferenceGetAlignmentSlice ( NGS_Reference * self,
- ctx_t ctx,
- uint64_t offset,
- uint64_t size,
- bool wants_primary,
+static struct NGS_Alignment* Null_ReferenceGetAlignmentSlice ( NGS_Reference * self,
+ ctx_t ctx,
+ uint64_t offset,
+ uint64_t size,
+ bool wants_primary,
bool wants_secondary,
uint32_t filters,
int32_t map_qual)
@@ -870,12 +888,18 @@ static bool Null_ReferenceIteratorNext ( NGS_Reference * self, ctx_t ctx )
return false;
}
+static struct NGS_ReferenceBlobIterator* Null_ReferenceGetBlobs ( const NGS_Reference * self, ctx_t ctx, uint64_t offset, uint64_t size )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcAccessing);
+ INTERNAL_ERROR ( xcSelfNull, "NULL Reference accessed" );
+ return NULL;
+}
static NGS_Reference_vt Null_Reference_vt_inst =
{
/* NGS_Refcount */
{ Null_ReferenceWhack },
-
+
/* NGS_Reference */
Null_ReferenceGetCommonName,
Null_ReferenceGetCanonicalName,
@@ -890,11 +914,12 @@ static NGS_Reference_vt Null_Reference_vt_inst =
Null_ReferenceGetPileups,
Null_ReferenceGetPileupSlice,
Null_ReferenceGetStatistics,
-
+ Null_ReferenceGetBlobs,
+
/* NGS_ReferenceIterator */
Null_ReferenceIteratorNext
};
-
+
struct NGS_Reference * NGS_ReferenceMakeNull ( ctx_t ctx, struct NGS_ReadCollection * coll )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
diff --git a/libs/ngs/NGS_Reference.h b/libs/ngs/NGS_Reference.h
index 3e02e24..a22ad22 100644
--- a/libs/ngs/NGS_Reference.h
+++ b/libs/ngs/NGS_Reference.h
@@ -72,7 +72,7 @@ extern struct NGS_Reference_v1_vt ITF_Reference_vt;
*/
#define NGS_ReferenceDuplicate( self, ctx ) \
( ( NGS_Reference* ) NGS_RefcountDuplicate ( NGS_ReferenceToRefcount ( self ), ctx ) )
-
+
/* GetCommonName
*/
struct NGS_String * NGS_ReferenceGetCommonName ( NGS_Reference * self, ctx_t ctx );
@@ -150,18 +150,22 @@ struct NGS_Statistics* NGS_ReferenceGetStatistics ( const NGS_Reference * self,
/*--------------------------------------------------------------------------
* NGS_ReferenceIterator
*/
-
+
/* Next
*/
bool NGS_ReferenceIteratorNext ( NGS_Reference * self, ctx_t ctx );
-
+
+/* FRAGMENT BLOBS
+ */
+struct NGS_ReferenceBlobIterator* NGS_ReferenceGetBlobs ( NGS_Reference * self, ctx_t ctx, uint64_t offset, uint64_t size );
+
/*--------------------------------------------------------------------------
* implementation details
*/
struct NGS_Reference
{
NGS_Refcount dad;
-
+
struct NGS_ReadCollection * coll;
};
@@ -169,7 +173,7 @@ typedef struct NGS_Reference_vt NGS_Reference_vt;
struct NGS_Reference_vt
{
NGS_Refcount_vt dad;
-
+
struct NGS_String * ( * get_common_name ) ( NGS_REFERENCE * self, ctx_t ctx );
struct NGS_String * ( * get_canonical_name ) ( NGS_REFERENCE * self, ctx_t ctx );
bool ( * get_is_circular ) ( const NGS_REFERENCE * self, ctx_t ctx );
@@ -186,22 +190,23 @@ struct NGS_Reference_vt
struct NGS_Pileup* ( * get_pileup_slice ) ( NGS_REFERENCE * self, ctx_t ctx, uint64_t offset, uint64_t size,
bool wants_primary, bool wants_secondary, uint32_t filters, int32_t map_qual );
struct NGS_Statistics* ( * get_statistics ) ( const NGS_REFERENCE * self, ctx_t ctx );
+ struct NGS_ReferenceBlobIterator* ( * get_blobs ) ( const NGS_REFERENCE * self, ctx_t ctx, uint64_t offset, uint64_t size );
bool ( * next ) ( NGS_REFERENCE * self, ctx_t ctx );
};
/* Init
*/
-void NGS_ReferenceInit ( ctx_t ctx,
- struct NGS_Reference * self,
- NGS_Reference_vt * vt,
- const char *clsname,
- const char *instname,
+void NGS_ReferenceInit ( ctx_t ctx,
+ struct NGS_Reference * self,
+ NGS_Reference_vt * vt,
+ const char *clsname,
+ const char *instname,
struct NGS_ReadCollection * coll );
-
+
/* Whack
-*/
+*/
void NGS_ReferenceWhack ( NGS_Reference * self, ctx_t ctx );
-
+
/* NullReference
* will error out on any call
*/
diff --git a/libs/ngs/NGS_ReferenceBlob.c b/libs/ngs/NGS_ReferenceBlob.c
new file mode 100644
index 0000000..54e55bb
--- /dev/null
+++ b/libs/ngs/NGS_ReferenceBlob.c
@@ -0,0 +1,296 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "NGS_ReferenceBlob.h"
+
+#include <ngs/itf/Refcount.h>
+
+#include <kfc/ctx.h>
+#include <kfc/except.h>
+#include <kfc/xc.h>
+
+#include <vdb/blob.h>
+
+#include "NGS_Cursor.h"
+#include "CSRA1_Reference.h"
+#include "VByteBlob.h"
+
+#include "../vdb/blob-priv.h"
+#include <../libs/vdb/page-map.h>
+
+const int64_t ChunkSize = 5000;
+
+struct NGS_ReferenceBlob
+{
+ NGS_Refcount dad;
+
+ const VBlob* blob;
+
+ int64_t refFirst; /* rowId of the first row in the reference/slice */
+ int64_t rowId; /* rowId of the first row in the blob */
+ uint64_t count; /* number of rows in the blob */
+
+ int64_t first; /* rowId of the first row in VBlob (may differ from rowId) */
+
+ const void* data; /* start of the first row */
+ uint64_t size; /* from the start of the first row until the end of the blob */
+};
+
+void
+NGS_ReferenceBlobWhack ( NGS_ReferenceBlob * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcDestroying );
+ if ( self != NULL )
+ {
+ VBlobRelease ( ( VBlob * ) self -> blob );
+ }
+}
+
+static NGS_Refcount_vt NGS_ReferenceBlob_vt =
+{
+ NGS_ReferenceBlobWhack
+};
+
+struct NGS_ReferenceBlob * NGS_ReferenceBlobMake ( ctx_t ctx, const NGS_Cursor* p_curs, int64_t p_firstRowId, int64_t p_refFirstRowId, int64_t p_refLastRowId )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcConstructing );
+ if ( p_curs == NULL )
+ {
+ INTERNAL_ERROR ( xcParamNull, "NULL cursor object" );
+ }
+ else if ( p_refFirstRowId < 1 )
+ {
+ INTERNAL_ERROR ( xcParamNull, "Invalid refFirstRowId: %li", p_refFirstRowId );
+ }
+ else if ( p_firstRowId < p_refFirstRowId )
+ {
+ INTERNAL_ERROR ( xcParamNull, "Invalid rowId: %li (less than refFirstRowId=%li)", p_firstRowId, p_refFirstRowId );
+ }
+ else
+ {
+ NGS_ReferenceBlob * ret = calloc ( 1, sizeof * ret );
+ if ( ret == NULL )
+ {
+ SYSTEM_ERROR ( xcNoMemory, "allocating NGS_ReferenceBlob" );
+ }
+ else
+ {
+ TRY ( NGS_RefcountInit ( ctx, & ret -> dad, & ITF_Refcount_vt . dad, & NGS_ReferenceBlob_vt, "NGS_ReferenceBlob", "" ) )
+ {
+ TRY ( ret -> blob = NGS_CursorGetVBlob ( p_curs, ctx, p_firstRowId, reference_READ ) )
+ {
+ ret -> refFirst = p_refFirstRowId;
+ ret -> rowId = p_firstRowId;
+ TRY ( VByteBlob_ContiguousChunk ( ret -> blob, ctx, ret -> rowId, p_refLastRowId - p_firstRowId + 1, &ret -> data, &ret -> size, false ) )
+ {
+ int64_t first;
+ uint64_t count;
+ rc_t rc = VBlobIdRange ( ret -> blob, & first, & count );
+ if ( rc == 0 )
+ {
+ assert ( first <= ret -> rowId );
+ ret -> first = first;
+ if ( p_refLastRowId != 0 && first + count > p_refLastRowId )
+ { /* cut off rows beyound p_refLastRowId */
+ ret -> count = p_refLastRowId + 1 - first;
+ }
+ else
+ {
+ ret -> count = count - ( ret -> rowId - first );
+ }
+ return ret;
+ }
+ INTERNAL_ERROR ( xcUnexpected, "VBlobIdRange() rc = %R", rc );
+ }
+ VBlobRelease ( ( VBlob * ) ret -> blob );
+ }
+ }
+ free ( ret );
+ }
+ }
+ return NULL;
+}
+
+void NGS_ReferenceBlobRowRange ( const struct NGS_ReferenceBlob * self, ctx_t ctx, int64_t * p_first, uint64_t * p_count )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
+
+ if ( self == NULL )
+ {
+ INTERNAL_ERROR ( xcParamNull, "bad object reference" );
+ }
+ else
+ {
+ if ( p_first != NULL )
+ {
+ *p_first = self -> rowId;
+ }
+ if ( p_count != NULL )
+ {
+ *p_count = self -> count;
+ }
+ }
+}
+
+void NGS_ReferenceBlobRelease ( struct NGS_ReferenceBlob * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
+ if ( self != NULL )
+ {
+ NGS_RefcountRelease ( & self -> dad, ctx );
+ }
+}
+
+NGS_ReferenceBlob * NGS_ReferenceBlobDuplicate ( struct NGS_ReferenceBlob * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
+ if ( self != NULL )
+ {
+ NGS_RefcountDuplicate ( & self -> dad, ctx );
+ }
+ return self;
+}
+
+const void* NGS_ReferenceBlobData ( const struct NGS_ReferenceBlob * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
+
+ if ( self == NULL )
+ {
+ INTERNAL_ERROR ( xcParamNull, "bad object reference" );
+ }
+ else
+ {
+ return self -> data;
+ }
+ return 0;
+}
+
+uint64_t NGS_ReferenceBlobSize ( const struct NGS_ReferenceBlob * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
+
+ if ( self == NULL )
+ {
+ INTERNAL_ERROR ( xcParamNull, "bad object reference" );
+ }
+ else
+ {
+ return self -> size;
+ }
+ return 0;
+}
+
+uint64_t NGS_ReferenceBlobUnpackedSize ( const struct NGS_ReferenceBlob * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
+ uint64_t ret = 0;
+ if ( self == NULL )
+ {
+ INTERNAL_ERROR ( xcParamNull, "bad object reference" );
+ }
+ else
+ {
+ PageMapIterator pmIt;
+ rc_t rc = PageMapNewIterator ( (const PageMap*)self->blob->pm, &pmIt, self -> rowId - self -> first, self -> count );
+ if ( rc != 0 )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "PageMapNewIterator() rc = %R", rc );
+ }
+ else
+ {
+ row_count_t repeat;
+ do
+ {
+ repeat = PageMapIteratorRepeatCount ( &pmIt );
+ ret += repeat * PageMapIteratorDataLength ( &pmIt );
+ }
+ while ( PageMapIteratorAdvance ( &pmIt, repeat ) );
+ }
+ }
+ return ret;
+}
+
+void NGS_ReferenceBlobResolveOffset ( const struct NGS_ReferenceBlob * self, ctx_t ctx, uint64_t p_inBlob, uint64_t* p_inReference, uint32_t* p_repeatCount, uint64_t* p_increment )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
+ if ( self == NULL )
+ {
+ INTERNAL_ERROR ( xcParamNull, "bad object reference" );
+ }
+ else if ( p_inBlob >= self -> size )
+ {
+ INTERNAL_ERROR ( xcParamNull, "offset %lu is out of range (0-%lu)", p_inBlob, self -> size );
+ }
+ else if ( p_inReference == NULL )
+ {
+ INTERNAL_ERROR ( xcParamNull, "NULL return parameter" );
+ }
+ else
+ {
+ PageMapIterator pmIt;
+ rc_t rc = PageMapNewIterator ( (const PageMap*)self->blob->pm, &pmIt, self -> rowId - self -> first, self -> count );
+ if ( rc != 0 )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "PageMapNewIterator() rc = %R", rc );
+ }
+ else
+ {
+ uint64_t inUnrolledBlob = 0; /* offset from the starting position of self->rowId if all repetitions in the blob were unrolled */
+ while ( true )
+ {
+ row_count_t repeat = PageMapIteratorRepeatCount ( &pmIt );
+ int64_t size = PageMapIteratorDataLength ( &pmIt );
+ elem_count_t offset = PageMapIteratorDataOffset ( &pmIt );
+ if (inUnrolledBlob == 0)
+ { /* the first offset is not always 0! */
+ inUnrolledBlob = offset;
+ }
+ assert ( size <= ChunkSize ); /* this may be the last chunk, shorter than ChunkSize */
+ if ( p_inBlob < offset + size )
+ {
+ /* assume all the prior chunks of this reference were ChunkSize long */
+ * p_inReference = ( self -> rowId - self -> refFirst ) * ChunkSize + inUnrolledBlob + p_inBlob % ChunkSize;
+ if ( p_repeatCount != NULL )
+ {
+ * p_repeatCount = repeat;
+ }
+ if ( p_increment != NULL )
+ {
+ * p_increment = repeat > 1 ? size : 0;
+ }
+ return;
+ }
+ if ( ! PageMapIteratorAdvance ( &pmIt, repeat ) )
+ {
+ INTERNAL_ERROR ( xcParamNull, "offset %lu is not found in (row=%li, count=%lu)", p_inBlob, self -> rowId, self -> count );
+ return;
+ }
+ inUnrolledBlob += repeat * size;
+ }
+ }
+ }
+}
diff --git a/libs/ngs/NGS_ReferenceBlob.h b/libs/ngs/NGS_ReferenceBlob.h
new file mode 100644
index 0000000..90411f3
--- /dev/null
+++ b/libs/ngs/NGS_ReferenceBlob.h
@@ -0,0 +1,100 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#ifndef _h_NGS_ReferenceBlob_
+#define _h_NGS_ReferenceBlob_
+
+typedef struct NGS_ReferenceBlob NGS_ReferenceBlob;
+#ifndef _h_ngs_refcount_
+#define NGS_REFCOUNT NGS_ReferenceBlob
+#include "NGS_Refcount.h"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct NGS_ReferenceBlob;
+struct NGS_Cursor;
+struct NGS_String;
+
+/*--------------------------------------------------------------------------
+ * NGS_ReferenceBlob
+ * Access to blobs in a REFERENCE.READ column
+ * reference counted
+ */
+
+/* Make
+ * create a blob containing the given row in the reference
+ * blobRowId - Id of the required row, becomes the first row of the blob
+ * refFirstId - Id of the first row of the reference containing the blob
+ * refLastId [ 0 OK ] - Id of the last row of the reference (or slice) containing the blob (blob will be cut off if it contains futher rows); 0 if no cutoff is needed
+ */
+struct NGS_ReferenceBlob * NGS_ReferenceBlobMake ( ctx_t ctx, const struct NGS_Cursor* curs, int64_t blobRowId, int64_t refFirstId, int64_t refLastId );
+
+/* Release
+ * release reference to the blob
+ */
+void NGS_ReferenceBlobRelease ( struct NGS_ReferenceBlob * self, ctx_t ctx );
+
+/* Duplicate
+ * duplicate reference to the blob
+ */
+NGS_ReferenceBlob * NGS_ReferenceBlobDuplicate ( struct NGS_ReferenceBlob * self, ctx_t ctx );
+
+/* RowRange
+ */
+void NGS_ReferenceBlobRowRange ( const struct NGS_ReferenceBlob * self, ctx_t ctx, int64_t * p_first, uint64_t * p_count );
+
+/* Data
+ * returns the blob's data buffer
+ */
+const void* NGS_ReferenceBlobData ( const struct NGS_ReferenceBlob * self, ctx_t ctx );
+
+/* Size
+ * returns the size the blob's data buffer
+ */
+uint64_t NGS_ReferenceBlobSize ( const struct NGS_ReferenceBlob * self, ctx_t ctx );
+
+/* Unpacked Size
+ * returns the unpacked size of the blob's data
+ */
+uint64_t NGS_ReferenceBlobUnpackedSize ( const struct NGS_ReferenceBlob * self, ctx_t ctx );
+
+/* ResolveOffset
+ * resolve offset inside the blob into offset on the reference
+ * inBlob - an offset into the blob
+ * inReference [ OUT ] - offset into the reference, corresponding to inBlob
+ * uint32_t* repeatCount [ OUT, NULL OK ] - the number of repetitions of the reference chunk the offset is inside of (1 if no repeats)
+ * uint64_t* increment [ OUT, NULL OK ] - size of the repeated chunk (0 if no repeats)
+ */
+void NGS_ReferenceBlobResolveOffset ( const struct NGS_ReferenceBlob * self, ctx_t ctx, uint64_t inBlob, uint64_t* inReference, uint32_t* repeatCount, uint64_t* increment );
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _h_NGS_ReferenceBlob_ */
diff --git a/libs/ngs/NGS_FragmentBlobIterator.c b/libs/ngs/NGS_ReferenceBlobIterator.c
similarity index 63%
copy from libs/ngs/NGS_FragmentBlobIterator.c
copy to libs/ngs/NGS_ReferenceBlobIterator.c
index 4baa0c1..223b871 100644
--- a/libs/ngs/NGS_FragmentBlobIterator.c
+++ b/libs/ngs/NGS_ReferenceBlobIterator.c
@@ -24,12 +24,14 @@
*
*/
-#include "NGS_FragmentBlobIterator.h"
+#include "NGS_ReferenceBlobIterator.h"
#include <kfc/ctx.h>
#include <kfc/except.h>
#include <kfc/xc.h>
+#include <klib/rc.h>
+
#include <ngs/itf/Refcount.h>
#include <vdb/cursor.h>
@@ -37,62 +39,58 @@
#include "NGS_String.h"
#include "NGS_Cursor.h"
#include "SRA_Read.h"
-#include "NGS_FragmentBlob.h"
+#include "NGS_ReferenceBlob.h"
-struct NGS_FragmentBlobIterator
+struct NGS_ReferenceBlobIterator
{
NGS_Refcount dad;
- const NGS_String* run;
const NGS_Cursor* curs;
- int64_t last_row;
+ int64_t ref_start;
int64_t next_row;
+ int64_t last_row;
};
void
-NGS_FragmentBlobIteratorWhack ( NGS_FragmentBlobIterator * self, ctx_t ctx )
+NGS_ReferenceBlobIteratorWhack ( NGS_ReferenceBlobIterator * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcDestroying );
if ( self != NULL )
{
NGS_CursorRelease ( self->curs, ctx );
- NGS_StringRelease ( self->run, ctx );
}
}
-static NGS_Refcount_vt NGS_FragmentBlobIterator_vt =
+static NGS_Refcount_vt NGS_ReferenceBlobIterator_vt =
{
- NGS_FragmentBlobIteratorWhack
+ NGS_ReferenceBlobIteratorWhack
};
-NGS_FragmentBlobIterator*
-NGS_FragmentBlobIteratorMake ( ctx_t ctx, const NGS_String* run, const struct VTable* tbl )
+NGS_ReferenceBlobIterator*
+NGS_ReferenceBlobIteratorMake ( ctx_t ctx, const struct NGS_Cursor* p_curs, int64_t p_refStartId, int64_t p_firstRowId, int64_t p_lastRowId )
{
FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcConstructing );
- if ( tbl == NULL )
+ if ( p_curs == NULL )
{
- INTERNAL_ERROR ( xcParamNull, "NULL table object" );
+ INTERNAL_ERROR ( xcParamNull, "NULL cursor object" );
}
else
{
- NGS_FragmentBlobIterator* ret = malloc ( sizeof * ret );
+ NGS_ReferenceBlobIterator* ret = malloc ( sizeof * ret );
if ( ret == NULL )
{
- SYSTEM_ERROR ( xcNoMemory, "allocating NGS_FragmentBlobIterator" );
+ SYSTEM_ERROR ( xcNoMemory, "allocating NGS_ReferenceBlobIterator" );
}
else
{
- TRY ( NGS_RefcountInit ( ctx, & ret -> dad, & ITF_Refcount_vt . dad, & NGS_FragmentBlobIterator_vt, "NGS_FragmentBlobIterator", "" ) )
+ TRY ( NGS_RefcountInit ( ctx, & ret -> dad, & ITF_Refcount_vt . dad, & NGS_ReferenceBlobIterator_vt, "NGS_ReferenceBlobIterator", "" ) )
{
- TRY ( ret -> curs = NGS_CursorMake ( ctx, tbl, sequence_col_specs, seq_NUM_COLS ) )
+ TRY ( ret -> curs = NGS_CursorDuplicate ( p_curs, ctx ) )
{
- TRY ( ret -> run = NGS_StringDuplicate ( run, ctx ) )
- {
- ret -> last_row = NGS_CursorGetRowCount ( ret->curs, ctx );
- ret -> next_row = 1;
- return ret;
- }
- NGS_CursorRelease ( ret -> curs, ctx );
+ ret -> ref_start = p_refStartId;
+ ret -> next_row = p_firstRowId;
+ ret -> last_row = p_lastRowId;
+ return ret;
}
}
free ( ret );
@@ -105,7 +103,7 @@ NGS_FragmentBlobIteratorMake ( ctx_t ctx, const NGS_String* run, const struct VT
* release reference
*/
void
-NGS_FragmentBlobIteratorRelease ( NGS_FragmentBlobIterator * self, ctx_t ctx )
+NGS_ReferenceBlobIteratorRelease ( NGS_ReferenceBlobIterator * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcReleasing );
if ( self != NULL )
@@ -117,45 +115,45 @@ NGS_FragmentBlobIteratorRelease ( NGS_FragmentBlobIterator * self, ctx_t ctx )
/* Duplicate
* duplicate reference
*/
-NGS_FragmentBlobIterator *
-NGS_FragmentBlobIteratorDuplicate ( NGS_FragmentBlobIterator * self, ctx_t ctx )
+NGS_ReferenceBlobIterator *
+NGS_ReferenceBlobIteratorDuplicate ( NGS_ReferenceBlobIterator * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
if ( self != NULL )
{
NGS_RefcountDuplicate( & self -> dad, ctx );
}
- return ( NGS_FragmentBlobIterator* ) self;
+ return ( NGS_ReferenceBlobIterator* ) self;
}
/* HasMore
* return true if there are more blobs to iterate on
*/
bool
-NGS_FragmentBlobIteratorHasMore ( NGS_FragmentBlobIterator * self, ctx_t ctx )
+NGS_ReferenceBlobIteratorHasMore ( NGS_ReferenceBlobIterator * self, ctx_t ctx )
{
if ( self == NULL )
{
FUNC_ENTRY ( ctx, rcSRA, rcRow, rcSelecting );
- INTERNAL_ERROR ( xcSelfNull, "NULL FragmentBlobIterator accessed" );
+ INTERNAL_ERROR ( xcSelfNull, "NULL ReferenceBlobIterator accessed" );
return false;
}
- return self != NULL && self -> next_row <= self -> last_row;
+ return self -> next_row <= self -> last_row;
}
/* Next
* return the next blob, to be Release()d by the caller.
* NULL if there are no more blobs
*/
-NGS_FragmentBlob*
-NGS_FragmentBlobIteratorNext ( NGS_FragmentBlobIterator * self, ctx_t ctx )
+NGS_ReferenceBlob*
+NGS_ReferenceBlobIteratorNext ( NGS_ReferenceBlobIterator * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
if ( self == NULL )
{
FUNC_ENTRY ( ctx, rcSRA, rcRow, rcSelecting );
- INTERNAL_ERROR ( xcSelfNull, "NULL FragmentBlobIterator accessed" );
+ INTERNAL_ERROR ( xcSelfNull, "NULL ReferenceBlobIterator accessed" );
}
else if ( self -> next_row <= self -> last_row )
{
@@ -166,18 +164,22 @@ NGS_FragmentBlobIteratorNext ( NGS_FragmentBlobIterator * self, ctx_t ctx )
& nextRow );
if ( rc == 0 )
{
- TRY ( NGS_FragmentBlob* ret = NGS_FragmentBlobMake ( ctx, self -> run, self -> curs, nextRow ) )
+ TRY ( NGS_ReferenceBlob* ret = NGS_ReferenceBlobMake ( ctx, self -> curs, nextRow, self -> ref_start, self -> last_row ) )
{
int64_t first;
uint64_t count;
- TRY ( NGS_FragmentBlobRowRange ( ret, ctx, & first, & count ) )
+ TRY ( NGS_ReferenceBlobRowRange ( ret, ctx, & first, & count ) )
{
self -> next_row = first + count;
return ret;
}
- NGS_FragmentBlobRelease ( ret, ctx );
+ NGS_ReferenceBlobRelease ( ret, ctx );
}
}
+ else if ( GetRCState ( rc ) != rcNotFound )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "VCursorFindNextRowIdDirect(READ, row=%li ) rc = %R", self -> next_row, rc );
+ }
self -> next_row = self -> last_row + 1;
}
diff --git a/libs/ngs/NGS_ReferenceBlobIterator.h b/libs/ngs/NGS_ReferenceBlobIterator.h
new file mode 100644
index 0000000..e689c24
--- /dev/null
+++ b/libs/ngs/NGS_ReferenceBlobIterator.h
@@ -0,0 +1,82 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#ifndef _h_NGS_ReferenceBlobIterator_
+#define _h_NGS_ReferenceBlobIterator_
+
+typedef struct NGS_ReferenceBlobIterator NGS_ReferenceBlobIterator;
+#ifndef _h_ngs_refcount_
+#define NGS_REFCOUNT NGS_ReferenceBlobIterator
+#include "NGS_Refcount.h"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct NGS_Cursor;
+struct NGS_ReferenceBlob;
+
+/*--------------------------------------------------------------------------
+ * NGS_ReferenceBlobIterator
+ * Iteration over blobs in the REFERENCE.READ column belonging to a single Reference sequence
+ *
+ * reference counted
+ */
+
+/* Make
+ * refStartId - Id of the fors row of the reference
+ * firstRowId - Id of the first row of the blob
+ * lastRowId - Id of the last row of the blob
+ */
+NGS_ReferenceBlobIterator* NGS_ReferenceBlobIteratorMake ( ctx_t ctx, const struct NGS_Cursor* curs, int64_t refStartId, int64_t firstRowId, int64_t lastRowId );
+
+/* Release
+ * release reference
+ */
+void NGS_ReferenceBlobIteratorRelease ( NGS_ReferenceBlobIterator * self, ctx_t ctx );
+
+/* Duplicate
+ * duplicate reference
+ */
+NGS_ReferenceBlobIterator * NGS_ReferenceBlobIteratorDuplicate ( NGS_ReferenceBlobIterator * self, ctx_t ctx );
+
+/* HasMore
+ * return true if there are more blobs to iterate on
+ */
+bool NGS_ReferenceBlobIteratorHasMore ( NGS_ReferenceBlobIterator * self, ctx_t ctx );
+
+/* Next
+ * return the next blob, to be Release()d by the caller.
+ * NULL if there are no more blobs
+ */
+struct NGS_ReferenceBlob* NGS_ReferenceBlobIteratorNext ( NGS_ReferenceBlobIterator * self, ctx_t ctx );
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _h_NGS_ReferenceBlob_ */
diff --git a/libs/ngs/SRA_DB_ReadCollection.c b/libs/ngs/SRA_DB_ReadCollection.c
index ef0bfaa..0d8da4e 100644
--- a/libs/ngs/SRA_DB_ReadCollection.c
+++ b/libs/ngs/SRA_DB_ReadCollection.c
@@ -177,7 +177,7 @@ NGS_Reference * SRA_DB_ReadCollectionGetReferences ( SRA_DB_ReadCollection * sel
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // create empty reference iterator
+ /* create empty reference iterator */
return NGS_ReferenceMakeNull ( ctx, & self -> dad );
}
@@ -192,7 +192,7 @@ NGS_Reference * SRA_DB_ReadCollectionGetReference ( SRA_DB_ReadCollection * self
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // always fail
+ /* always fail */
INTERNAL_ERROR ( xcRowNotFound, "Reference not found ( NAME = %s )", spec );
return NULL;
}
@@ -203,7 +203,7 @@ NGS_Alignment * SRA_DB_ReadCollectionGetAlignments ( SRA_DB_ReadCollection * sel
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // create empty alignment iterator
+ /* create empty alignment iterator */
return NGS_AlignmentMakeNull ( ctx, NGS_StringData(self -> run_name, ctx), NGS_StringSize(self -> run_name, ctx) );
}
@@ -212,7 +212,7 @@ NGS_Alignment * SRA_DB_ReadCollectionGetAlignment ( SRA_DB_ReadCollection * self
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // always fail
+ /* always fail */
INTERNAL_ERROR ( xcRowNotFound, "Aligment not found ( ID = %ld )", alignmentId );
return NULL;
}
@@ -230,7 +230,7 @@ NGS_Alignment * SRA_DB_ReadCollectionGetAlignmentRange ( SRA_DB_ReadCollection *
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // create empty alignment iterator
+ /* create empty alignment iterator */
return NGS_AlignmentMakeNull ( ctx, NGS_StringData(self -> run_name, ctx), NGS_StringSize(self -> run_name, ctx) );
}
diff --git a/libs/ngs/SRA_Read.c b/libs/ngs/SRA_Read.c
index 32cef46..6641687 100644
--- a/libs/ngs/SRA_Read.c
+++ b/libs/ngs/SRA_Read.c
@@ -66,6 +66,7 @@ const char * sequence_col_specs [] =
"SPOT_GROUP",
"PRIMARY_ALIGNMENT_ID",
"(U64)SPOT_COUNT",
+ "(INSDC:dna:text)CMP_READ",
};
static NGS_Read_vt NGS_Read_vt_inst =
diff --git a/libs/ngs/SRA_Read.h b/libs/ngs/SRA_Read.h
index e5fc797..3583eac 100644
--- a/libs/ngs/SRA_Read.h
+++ b/libs/ngs/SRA_Read.h
@@ -52,7 +52,7 @@ struct NGS_Cursor;
struct NGS_String;
enum SequenceTableColumn
-{ // keep in sync with sequence_col_specs in SRA_Read.c
+{ /* keep in sync with sequence_col_specs in SRA_Read.c */
seq_READ,
seq_READ_TYPE,
seq_QUALITY,
@@ -61,9 +61,10 @@ enum SequenceTableColumn
seq_GROUP,
seq_PRIMARY_ALIGNMENT_ID,
seq_SPOT_COUNT,
+ seq_CMP_READ,
seq_NUM_COLS
-}; // keep in sync with sequence_col_specs in SRA_Read.c
+}; /* keep in sync with sequence_col_specs in SRA_Read.c */
extern const char * sequence_col_specs [];
diff --git a/libs/ngs/SRA_ReadCollection.c b/libs/ngs/SRA_ReadCollection.c
index 355dc71..d907960 100644
--- a/libs/ngs/SRA_ReadCollection.c
+++ b/libs/ngs/SRA_ReadCollection.c
@@ -165,7 +165,7 @@ NGS_Reference * SRA_ReadCollectionGetReferences ( SRA_ReadCollection * self, ctx
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // create empty reference iterator
+ /* create empty reference iterator */
return NGS_ReferenceMakeNull ( ctx, & self -> dad );
}
@@ -180,7 +180,7 @@ NGS_Reference * SRA_ReadCollectionGetReference ( SRA_ReadCollection * self, ctx_
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // always fail
+ /* always fail */
INTERNAL_ERROR ( xcRowNotFound, "Reference not found ( NAME = %s )", spec );
return NULL;
}
@@ -191,7 +191,7 @@ NGS_Alignment * SRA_ReadCollectionGetAlignments ( SRA_ReadCollection * self, ctx
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // create empty alignment iterator
+ /* create empty alignment iterator */
return NGS_AlignmentMakeNull ( ctx, NGS_StringData(self -> run_name, ctx), NGS_StringSize(self -> run_name, ctx) );
}
@@ -200,7 +200,7 @@ NGS_Alignment * SRA_ReadCollectionGetAlignment ( SRA_ReadCollection * self, ctx_
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // always fail
+ /* always fail */
INTERNAL_ERROR ( xcRowNotFound, "Aligment not found ( ID = %ld )", alignmentId );
return NULL;
}
@@ -220,7 +220,7 @@ NGS_Alignment * SRA_ReadCollectionGetAlignmentRange ( SRA_ReadCollection * self,
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcAccessing );
- // create empty alignment iterator
+ /* create empty alignment iterator */
return NGS_AlignmentMakeNull ( ctx, NGS_StringData(self -> run_name, ctx), NGS_StringSize(self -> run_name, ctx) );
}
diff --git a/libs/ngs/VByteBlob.c b/libs/ngs/VByteBlob.c
index 80e5e79..e705139 100644
--- a/libs/ngs/VByteBlob.c
+++ b/libs/ngs/VByteBlob.c
@@ -37,7 +37,7 @@
* starts at rowId, ends before a repeated value or at the end of the blob
*/
void
-VByteBlob_ContiguousChunk ( const VBlob* p_blob, ctx_t ctx, int64_t rowId, const void** p_data, uint64_t* p_size, bool p_stopAtRepeat )
+VByteBlob_ContiguousChunk ( const VBlob* p_blob, ctx_t ctx, int64_t rowId, uint64_t p_maxRows, const void** p_data, uint64_t* p_size, bool p_stopAtRepeat )
{
FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
@@ -83,7 +83,7 @@ VByteBlob_ContiguousChunk ( const VBlob* p_blob, ctx_t ctx, int64_t rowId, cons
assert ( rowId >= first && rowId < first + (int64_t)count );
if ( rowId - first + 1 < (int64_t)count ) /* more rows in the blob */
- { /* *p_size is the size of value on rowId. Increase size to include subsequent rows, until we see a repeat or the blob ends */
+ { /* *p_size is the size of value on rowId. Increase size to include subsequent rows, until we see a repeat, p_maxRows is reached or the blob ends */
rc = PageMapNewIterator ( (const PageMap*)p_blob->pm, &pmIt, rowId - first, count - ( rowId - first ) ); /* here, rowId is relative to the blob */
if ( rc != 0 )
{
@@ -93,11 +93,25 @@ VByteBlob_ContiguousChunk ( const VBlob* p_blob, ctx_t ctx, int64_t rowId, cons
{
do
{
+ row_count_t repeat;
*p_size += PageMapIteratorDataLength ( &pmIt );
- if ( PageMapIteratorRepeatCount ( &pmIt ) > 1 )
+ repeat = PageMapIteratorRepeatCount ( &pmIt );
+ if ( p_maxRows != 0 )
{
+ if ( repeat < p_maxRows )
+ {
+ p_maxRows -= repeat;
+ }
+ else
+ { /* p_maxRows reached */
+ break;
+ }
+ }
+ if ( PageMapIteratorRepeatCount ( &pmIt ) > 1 )
+ { /* repeated row found */
break;
}
+
}
while ( PageMapIteratorNext ( &pmIt ) );
}
@@ -107,6 +121,17 @@ VByteBlob_ContiguousChunk ( const VBlob* p_blob, ctx_t ctx, int64_t rowId, cons
*p_size = row_len;
}
}
+ else if ( p_maxRows > 0 && p_maxRows < count - ( rowId - first ) )
+ { /* return the size of the first p_maxRows rows */
+ const uint8_t* firstRow = (const uint8_t*)base;
+ rc = VBlobCellData ( p_blob,
+ rowId + p_maxRows,
+ & elem_bits,
+ & base,
+ & boff,
+ & row_len );
+ *p_size = (const uint8_t*)( base ) - firstRow;
+ }
else
{ /* set the size to include the rest of the blob's data */
*p_size = BlobBufferBytes ( p_blob ) - ( (const uint8_t*)( base ) - (const uint8_t*)( p_blob -> data . base ) );
diff --git a/libs/ngs/VByteBlob.h b/libs/ngs/VByteBlob.h
index e9d623b..b7d6096 100644
--- a/libs/ngs/VByteBlob.h
+++ b/libs/ngs/VByteBlob.h
@@ -36,9 +36,9 @@ extern "C" {
#endif
/* Calculate the biggest available contiguous data portion of the blob:
-* starts at rowId, ends before a repeated value or at the end of the blob
+* starts at rowId, ends at MaxRows if not 0, before a repeated value or at the end of the blob
*/
-void VByteBlob_ContiguousChunk ( const struct VBlob* blob, ctx_t ctx, int64_t rowId, const void** data, uint64_t* size, bool stopAtRepeat );
+void VByteBlob_ContiguousChunk ( const struct VBlob* blob, ctx_t ctx, int64_t rowId, uint64_t maxRows, const void** data, uint64_t* size, bool stopAtRepeat );
#ifdef __cplusplus
}
diff --git a/libs/tui/tui_dlg_helper.c b/libs/tui/tui_dlg_helper.c
index 8553ae8..b0e37c7 100644
--- a/libs/tui/tui_dlg_helper.c
+++ b/libs/tui/tui_dlg_helper.c
@@ -30,6 +30,7 @@
#include <klib/text.h>
#include <klib/printf.h>
#include <kfs/directory.h>
+#include <kfs/filetools.h>
#include <vfs/manager.h>
#include <vfs/path.h>
@@ -137,32 +138,6 @@ static rc_t add_str_to_listbox( dir_callback_ctx * dctx, const char * name )
}
-static rc_t CC on_dir_entry( const KDirectory *dir, uint32_t type, const char * name, void * data )
-{
- rc_t rc = 0;
- if ( name != NULL && data != NULL && name[ 0 ] != 0 && name[ 0 ] != '.' )
- {
- dir_callback_ctx * dctx = data;
- if ( ( type & ~kptAlias ) == kptDir )
- {
- rc = add_str_to_listbox( dctx, name );
- if ( rc == 0 )
- {
- if ( dctx->str_to_focus != NULL )
- {
- int cmp = string_cmp ( name, string_size( name ),
- dctx->str_to_focus, string_size( dctx->str_to_focus ), 4096 );
- if ( cmp == 0 )
- dctx->to_focus = dctx->string_id;
- }
- dctx->string_id++;
- }
- }
- }
- return rc;
-}
-
-
rc_t fill_widget_with_dirs( struct KTUIDlg * dlg, KDirectory * dir, uint32_t id, const char * path, const char * to_focus )
{
rc_t rc;
@@ -172,6 +147,8 @@ rc_t fill_widget_with_dirs( struct KTUIDlg * dlg, KDirectory * dir, uint32_t id,
dctx.to_focus = 0;
dctx.str_to_focus = to_focus;
dctx.in_root = ( ( path[ 0 ] == '/' )&&( path[ 1 ] == 0 ) );
+ dctx.dlg = dlg;
+ dctx.widget_id = id;
rc = KTUIDlgRemoveAllWidgetStrings ( dlg, id );
if ( rc == 0 )
@@ -185,13 +162,37 @@ rc_t fill_widget_with_dirs( struct KTUIDlg * dlg, KDirectory * dir, uint32_t id,
if ( rc == 0 )
{
- dctx.dlg = dlg;
- dctx.widget_id = id;
+ VNamelist * dir_list;
+ rc = ReadDirEntriesIntoToNamelist( &dir_list, dir, true, false, true, path );
+ if ( rc == 0 )
+ {
+ uint32_t idx, count;
+ rc = VNameListCount( dir_list, &count );
+ for ( idx = 0; rc == 0 && idx < count; ++idx )
+ {
+ const char * name = NULL;
+ rc = VNameListGet( dir_list, idx, &name );
+ if ( rc == 0 && name != NULL )
+ {
+ rc = add_str_to_listbox( &dctx, name );
+ if ( rc == 0 )
+ {
+ if ( dctx.str_to_focus != NULL )
+ {
+ int cmp = string_cmp ( name, string_size( name ),
+ dctx.str_to_focus, string_size( dctx.str_to_focus ), 4096 );
+ if ( cmp == 0 )
+ dctx.to_focus = dctx.string_id;
+ }
+ dctx.string_id++;
+ }
+ }
+ }
+ VNamelistRelease( dir_list );
+ }
- /* we allow it to fail... */
- KDirectoryVisit ( dir, false, on_dir_entry, &dctx, "%s", path );
}
-
+
if ( rc == 0 && dctx.to_focus > 0 )
rc = KTUIDlgSetWidgetSelectedString ( dlg, id, dctx.to_focus );
@@ -199,52 +200,41 @@ rc_t fill_widget_with_dirs( struct KTUIDlg * dlg, KDirectory * dir, uint32_t id,
}
-typedef struct file_callback_ctx
-{
- struct KTUIDlg * dlg;
- const char * extension;
- uint32_t id;
-} file_callback_ctx;
-
-
-static rc_t CC on_file_entry( const KDirectory *dir, uint32_t type, const char * name, void * data )
+rc_t fill_widget_with_files( struct KTUIDlg * dlg, KDirectory * dir, uint32_t id, const char * path, const char * extension )
{
- rc_t rc = 0;
- if ( name != NULL && data != NULL && name[ 0 ] != 0 && name[ 0 ] != '.' )
+ rc_t rc = KTUIDlgRemoveAllWidgetStrings ( dlg, id );
+ if ( rc == 0 )
{
- file_callback_ctx * fctx = data;
- if ( ( type & ~kptAlias ) == kptFile )
+ VNamelist * file_list;
+ rc = ReadDirEntriesIntoToNamelist( &file_list, dir, true, true, false, path );
+ if ( rc == 0 )
{
- bool add = ( fctx->extension == NULL );
- if ( !add )
+ uint32_t idx, count;
+ rc = VNameListCount( file_list, &count );
+ for ( idx = 0; rc == 0 && idx < count; ++idx )
{
- size_t name_size = string_size ( name );
- size_t ext_size = string_size ( fctx->extension );
- if ( name_size > ext_size )
+ const char * name = NULL;
+ rc = VNameListGet( file_list, idx, &name );
+ if ( rc == 0 && name != NULL && name[ 0 ] != 0 && name[ 0 ] != '.' )
{
- int cmp = string_cmp ( &name[ name_size - ext_size ], ext_size,
- fctx->extension, ext_size, (uint32_t)ext_size );
- add = ( cmp == 0 );
+ bool add = ( extension == NULL );
+ if ( !add )
+ {
+ size_t name_size = string_size ( name );
+ size_t ext_size = string_size ( extension );
+ if ( name_size > ext_size )
+ {
+ int cmp = string_cmp ( &name[ name_size - ext_size ], ext_size,
+ extension, ext_size, (uint32_t)ext_size );
+ add = ( cmp == 0 );
+ }
+ }
+ if ( add )
+ rc = KTUIDlgAddWidgetString( dlg, id, name );
}
}
- if ( add )
- rc = KTUIDlgAddWidgetString ( fctx->dlg, fctx->id, name );
+ VNamelistRelease( file_list );
}
}
return rc;
}
-
-
-rc_t fill_widget_with_files( struct KTUIDlg * dlg, KDirectory * dir, uint32_t id, const char * path, const char * extension )
-{
- rc_t rc = KTUIDlgRemoveAllWidgetStrings ( dlg, id );
- if ( rc == 0 )
- {
- file_callback_ctx fctx;
- fctx.extension = extension;
- fctx.dlg = dlg;
- fctx.id = id;
- rc = KDirectoryVisit ( dir, false, on_file_entry, &fctx, "%s", path );
- }
- return rc;
-}
diff --git a/libs/vdb/cursor-cmn.c b/libs/vdb/cursor-cmn.c
index d5c5188..18d6c95 100644
--- a/libs/vdb/cursor-cmn.c
+++ b/libs/vdb/cursor-cmn.c
@@ -2816,7 +2816,7 @@ rc_t VCursorFindNextRowIdInt ( const VCursor * self, uint32_t idx, int64_t start
bool is_static = false;
KColumn * kcol = NULL;
rc = VColumnGetKColumn ( vcol, & kcol, & is_static );
- if ( rc == 0 )
+ if ( kcol != NULL && rc == 0 )
{
/* we have a physical column - ask kdb what the next id is */
assert ( kcol != NULL );
diff --git a/libs/vdb/libvdb.vers.h b/libs/vdb/libvdb.vers.h
index 734f137..db38b84 100644
--- a/libs/vdb/libvdb.vers.h
+++ b/libs/vdb/libvdb.vers.h
@@ -24,4 +24,4 @@
*
*/
-#define LIBVDB_VERS 0x02070016
+#define LIBVDB_VERS 0x02070017
diff --git a/libs/vfs/Makefile b/libs/vfs/Makefile
index 81c251c..3812fc5 100644
--- a/libs/vfs/Makefile
+++ b/libs/vfs/Makefile
@@ -73,10 +73,13 @@ clean: stdclean
$(ILIBDIR)/libvfs: $(addprefix $(ILIBDIR)/libvfs.,$(ILIBEXT))
VFS_SRC_CMN = \
- syspath \
manager \
+ remote-services \
resolver \
resolver-3.0 \
+ services \
+ srv-response \
+ syspath \
VFS_SRC = \
path \
diff --git a/libs/vfs/manager.c b/libs/vfs/manager.c
index d85f195..6dc6ec9 100644
--- a/libs/vfs/manager.c
+++ b/libs/vfs/manager.c
@@ -1416,7 +1416,6 @@ rc_t TransformFileToDirectory(const KDirectory * dir,
return rc;
}
-/* also handles ftp - if it cant we'll need another function */
static
rc_t VFSManagerOpenDirectoryReadHttp (const VFSManager *self,
const KDirectory * dir,
@@ -1441,13 +1440,13 @@ rc_t VFSManagerOpenDirectoryReadHttp (const VFSManager *self,
extension, sizeof extension - 1, sizeof extension - 1 ) != 0 )
{
const String * p = NULL;
- rc_t rc = VPathMakeString ( path, & p );
- if ( rc == 0 ) {
- PLOGERR ( klogErr, ( klogErr, rc, "error with http open '$(path)'",
+ rc_t rc2 = VPathMakeString ( path, & p );
+ if ( rc2 == 0 ) {
+ PLOGERR ( klogErr, ( klogErr, rc, "error with https open '$(path)'",
"path=%S", p ) );
free ( ( void * ) p );
} else {
- PLOGERR ( klogErr, ( klogErr, rc, "error with http open '$(scheme):$(path)'",
+ PLOGERR ( klogErr, ( klogErr, rc, "error with https open '$(scheme):$(path)'",
"scheme=%S,path=%S", & path -> scheme, s ) );
}
}
@@ -1524,7 +1523,7 @@ rc_t VFSManagerOpenDirectoryReadHttpResolved (const VFSManager *self,
{
if ( high_reliability )
{
- PLOGERR ( klogErr, ( klogErr, rc, "error with http open '$(U)'",
+ PLOGERR ( klogErr, ( klogErr, rc, "error with https open '$(U)'",
"U=%S", uri ) );
}
}
@@ -1882,6 +1881,8 @@ LIB_EXPORT rc_t CC VFSManagerOpenDirectoryRead (const VFSManager *self,
KDirectory const **d,
const VPath * path)
{
+ if ( self == NULL )
+ return RC (rcVFS, rcDirectory, rcOpening, rcSelf, rcNull);
return VFSManagerOpenDirectoryReadDirectoryRelativeInt (self, self->cwd, d, path, false);
}
@@ -2384,7 +2385,7 @@ LIB_EXPORT rc_t CC VFSManagerMakeFromKfg ( struct VFSManager ** pmanager,
kfsmanager_classname, "init", "singleton" );
/* hard-coded default */
- obj -> protocols = eProtocolHttpHttps;
+ obj -> protocols = DEFAULT_PROTOCOLS;
rc = KDirectoryNativeDir ( & obj -> cwd );
if ( rc == 0 )
@@ -3410,6 +3411,14 @@ LIB_EXPORT rc_t CC VFSManagerSetCacheRoot ( const VFSManager * self,
rc = VPathMakeString ( path, &spath );
if ( rc == 0 )
{
+ /* in case the path ends in a '/' ( the path-separator ) we have to remove it... */
+ if ( spath->addr[ spath->len - 1 ] == '/' )
+ {
+ String * p = ( String * )spath;
+ p->len -= 1;
+ p->size -= 1;
+ ( ( char * )p->addr )[ p->len ] = 0;
+ }
rc = KConfigWriteSString( self -> cfg, default_path_key, spath );
StringWhack( spath );
/*
@@ -3469,6 +3478,21 @@ static rc_t inspect_dir( KDirectory * dir, KTime_t date, const char * path )
}
KNamelistRelease( itemlist );
}
+ else
+ {
+ if ( ( GetRCModule( rc ) == rcFS ) &&
+ ( GetRCTarget( rc ) == rcDirectory ) &&
+ ( GetRCContext( rc ) == rcListing ) &&
+ ( GetRCObject( rc ) == ( enum RCObject )rcPath ) &&
+ ( GetRCState( rc ) == rcNotFound ) )
+ {
+ rc = 0;
+ }
+ else
+ {
+ PLOGERR( klogErr, ( klogErr, rc, "KDirectoryList( '$(P)' )", "P=%s", path ) );
+ }
+ }
return rc;
}
diff --git a/libs/vfs/path-priv.h b/libs/vfs/path-priv.h
index 902842b..9fcfb31 100644
--- a/libs/vfs/path-priv.h
+++ b/libs/vfs/path-priv.h
@@ -47,6 +47,10 @@
#include <vfs/path.h>
#endif
+#ifndef _h_vfs_resolver_
+#include <vfs/resolver.h> /* VRemoteProtocols */
+#endif
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -54,6 +58,14 @@ extern "C" {
/*--------------------------------------------------------------------------
* VPath
*/
+/* VPath Type:
+ * how many extended properties ( from name resolver response ) are initialized
+ */
+typedef enum {
+ eVPnoExt, /* does not have extended part */
+ eVPWithId, /* has object-id */
+ eVPext, /* has all extanded properties */
+} EVPathType;
struct VPath
{
KDataBuffer data;
@@ -82,6 +94,21 @@ struct VPath
bool from_uri;
bool missing_port;
bool highly_reliable;
+
+ /* how many extended properties ( from name resolver response )
+ are initialized */
+ EVPathType ext;
+
+ String id; /* object-id */
+
+ String tick; /* dbGaP download ticket */
+ size_t size; /* object's un-encrypted size in byte */
+ KTime_t modification; /* object's modification date. 0 if unknown */
+ KTime_t expiration; /* expiration date of this VPath object.
+ 0 if infinite */
+
+ uint8_t md5 [ 16 ]; /* md5 checksum object's un-encrypted if known */
+ bool has_md5;
};
enum VPathVariant
@@ -141,8 +168,60 @@ VFS_EXTERN rc_t CC VPathGetScheme_t ( const VPath * self, VPUri_t * uri_type );
VPUri_t VPathGetUri_t (const VPath * self);
+rc_t VPathMakeFromUrl ( VPath ** new_path, const String * url,
+ const String * tick, bool ext, const String * id, size_t size, KTime_t date,
+ const uint8_t md5 [ 16 ], KTime_t exp_date );
+
+/* Equal
+ * compares two VPath-s
+ *
+ * "notequal" [ OUT ] - is set
+ * to union of bits corresponding to difference in different VPath properties
+ *
+ * returns non-0 rc after a failed call to get property from any of VPath-s
+ */
+rc_t VPathEqual ( const VPath * l, const VPath * r, int * notequal );
+/* Close
+ * compares two VPath-s
+ * difference between expirations should be withing expirationRange */
+rc_t VPathClose ( const VPath * l, const VPath * r, int * notequal,
+ KTime_t expirationRange );
+
+
+/***** VPathSet - set of VPath's - genetated from name resolver response ******/
+
+typedef struct VPathSet VPathSet;
+
+rc_t VPathSetRelease ( const VPathSet * self );
+rc_t VPathSetGet ( const VPathSet * self, VRemoteProtocols protocols,
+ const struct VPath ** path, const struct VPath ** vdbcache );
+
+/* name resolver response row converted into VDB objects */
+typedef struct {
+ struct VPath * fasp ; struct VPath * vcFasp;
+ struct VPath * file ; struct VPath * vcFile;
+ struct VPath * http ; struct VPath * vcHttp;
+ struct VPath * https; struct VPath * vcHttps;
+ struct VPath * s3 ; struct VPath * vcS3;
+ struct VPath * mapping;
+ const struct KSrvError * error;
+} EVPath;
+
+rc_t VPathSetMake
+ ( VPathSet ** self, const EVPath * src, bool singleUrl );
+
+rc_t VPathSetMakeQuery ( VPathSet ** self, const VPath * local, rc_t localRc,
+ const VPath * cache, rc_t cacheRc );
+
#ifdef __cplusplus
}
#endif
#endif /* _h_path_priv_ */
+
+#if 0
+/******************************** KSrvResponse ********************************/
+rc_t KSrvResponseRelease ( const KSrvResponse * self );
+uint32_t KSrvResponseLength ( const KSrvResponse * self );
+/******************************************************************************/
+#endif
diff --git a/libs/vfs/path.c b/libs/vfs/path.c
index 240a289..b553a08 100644
--- a/libs/vfs/path.c
+++ b/libs/vfs/path.c
@@ -57,6 +57,9 @@ void VPathWhack ( VPath * self )
{
KDataBufferWhack ( & self -> data );
KRefcountWhack ( & self -> refcount, "VPath" );
+ free ( ( void * ) self -> id . addr );
+ free ( ( void * ) self -> tick . addr );
+ memset ( self, 0, sizeof * self );
free ( self );
}
@@ -3288,6 +3291,73 @@ LIB_EXPORT uint32_t CC VPathGetOid ( const VPath * self )
}
+LIB_EXPORT rc_t CC VPathGetId ( const VPath * self, String * str )
+{
+ rc_t rc;
+
+ if ( str == NULL )
+ rc = RC ( rcVFS, rcPath, rcAccessing, rcParam, rcNull );
+ else
+ {
+ rc = VPathGetTestSelf ( self );
+ if ( rc == 0 )
+ {
+ * str = self -> id;
+ return 0;
+ }
+
+ StringInit ( str, "", 0, 0 );
+ }
+
+ return rc;
+}
+
+LIB_EXPORT rc_t CC VPathGetTicket ( const VPath * self, String * str )
+{
+ rc_t rc;
+
+ if ( str == NULL )
+ rc = RC ( rcVFS, rcPath, rcAccessing, rcParam, rcNull );
+ else
+ {
+ rc = VPathGetTestSelf ( self );
+ if ( rc == 0 )
+ {
+ * str = self -> tick;
+ return 0;
+ }
+
+ StringInit ( str, "", 0, 0 );
+ }
+
+ return rc;
+}
+
+LIB_EXPORT KTime_t CC VPathGetModDate ( const VPath * self )
+{
+ if ( self != NULL )
+ return self -> modification;
+ return 0;
+}
+
+LIB_EXPORT size_t CC VPathGetSize ( const VPath * self )
+{
+ if ( self != NULL )
+ return self -> size;
+ return 0;
+}
+
+LIB_EXPORT const uint8_t * CC VPathGetMd5 ( const VPath * self )
+{
+ if ( self == NULL )
+ return NULL;
+ if ( ! self -> has_md5 )
+ return NULL;
+
+ return self -> md5;
+}
+
+
/* MarkHighReliability
* mark a path as representing either a reliable URL
* or one where the reliability is unknown.
@@ -3727,7 +3797,10 @@ rc_t LegacyVPathMakeFmt ( VPath ** new_path, const char * fmt, ... )
return rc;
}
-rc_t LegacyVPathMakeVFmt ( VPath ** new_path, const char * fmt, va_list args )
+static
+rc_t VPathMakeVFmtExt ( EVPathType ext, VPath ** new_path, const String * id,
+ const String * tick, size_t size, KTime_t date, const uint8_t md5 [ 16 ],
+ KTime_t exp_date, const char * fmt, va_list args )
{
rc_t rc;
@@ -3754,6 +3827,36 @@ rc_t LegacyVPathMakeVFmt ( VPath ** new_path, const char * fmt, va_list args )
path -> scheme = scheme;
}
+ path -> ext = ext;
+ path -> size = size;
+ path -> modification = date;
+ path -> expiration = exp_date;
+
+ if ( md5 != NULL ) {
+ int i = 0;
+ for ( i = 0; i < 16; ++ i )
+ path -> md5 [ i ] = md5 [ i ];
+ path -> has_md5 = true;
+ }
+
+ if ( id != NULL && id -> size > 0 ) {
+ StringInit ( & path -> id,
+ string_dup ( id -> addr, id -> size ),
+ id -> size, id -> len );
+ if ( path -> id . addr == NULL )
+ return RC ( rcVFS,
+ rcPath, rcAllocating, rcMemory, rcExhausted );
+ }
+
+ if ( tick != NULL && tick -> size > 0 ) {
+ StringInit ( & path -> tick,
+ string_dup ( tick -> addr, tick -> size ),
+ tick -> size, tick -> len );
+ if ( path -> tick . addr == NULL )
+ return RC ( rcVFS,
+ rcPath, rcAllocating, rcMemory, rcExhausted );
+ }
+
return 0;
}
}
@@ -3764,6 +3867,43 @@ rc_t LegacyVPathMakeVFmt ( VPath ** new_path, const char * fmt, va_list args )
return rc;
}
+static
+rc_t VPathMakeFmtExt ( VPath ** new_path, bool ext, const String * id,
+ const String * tick, size_t size, KTime_t date, const uint8_t md5 [ 16 ],
+ KTime_t exp_date, const char * fmt, ... )
+{
+ EVPathType t = ext ? eVPext : eVPWithId;
+ rc_t rc;
+
+ va_list args;
+ va_start ( args, fmt );
+
+ rc = VPathMakeVFmtExt ( t, new_path, id, tick, size, date, md5, exp_date,
+ fmt, args );
+
+ va_end ( args );
+
+ return rc;
+}
+
+rc_t VPathMakeFromUrl ( VPath ** new_path, const String * url,
+ const String * tick, bool ext, const String * id, size_t size, KTime_t date,
+ const uint8_t md5 [ 16 ], KTime_t exp_date )
+{
+ if ( tick == NULL || tick -> addr == NULL || tick -> size == 0 )
+ return VPathMakeFmtExt ( new_path, ext, id, tick, size, date, md5,
+ exp_date, "%S", url );
+ else
+ return VPathMakeFmtExt ( new_path, ext, id, tick, size, date, md5,
+ exp_date, "%S?tic=%S", url, tick );
+}
+
+rc_t LegacyVPathMakeVFmt ( VPath ** new_path, const char * fmt, va_list args )
+{
+ return VPathMakeVFmtExt ( false, new_path, NULL, NULL, 0, 0, NULL, 0, fmt,
+ args );
+}
+
LIB_EXPORT rc_t CC LegacyVPathGetScheme_t ( const VPath * self, VPUri_t * uri_type )
{
rc_t rc = 0;
@@ -3799,3 +3939,151 @@ VPUri_t LegacyVPathGetUri_t ( const VPath * self )
LegacyVPathGetScheme_t ( self, & uri_type );
return uri_type;
}
+
+rc_t VPathClose ( const VPath * l, const VPath * r, int * notequal,
+ KTime_t expirationRange )
+{
+ rc_t rc = 0;
+ char pbuffer [ 999 ] = "";
+ size_t pnumred = 0;
+ char ebuffer [ 999 ] = "";
+ size_t end = 0;
+ rc_t rp = 0;
+ rc_t re = 0;
+ String pstr = { 0, 0, 0 };
+ String estr = { 0, 0, 0 };
+ VPUri_t puri_type;
+ VPUri_t euri_type;
+ int dummy = 0;
+ if ( notequal == NULL )
+ notequal = & dummy;
+
+ * notequal = 0;
+
+ if ( l == r ) {
+ return 0;
+ }
+
+ if ( ( l == NULL && r != NULL ) || ( r == NULL && l != NULL ) ) {
+ * notequal = 1;
+ return 0;
+ }
+
+ rp = VPathReadUri ( l, pbuffer, sizeof pbuffer, & pnumred );
+ re = VPathReadUri ( r, ebuffer, sizeof ebuffer, & end );
+ if ( rp == 0 && re == 0 ) {
+ if ( pnumred != end )
+ * notequal |= 2;
+ else if ( string_cmp ( pbuffer, pnumred, ebuffer, end, end ) != 0 )
+ * notequal |= 4;
+ }
+ else if ( rc == 0 ) {
+ if ( rp != 0 )
+ rc = rp;
+ else
+ rc = re;
+ }
+
+ rp = VPathGetAuth ( l, & pstr );
+ re = VPathGetAuth ( r, & estr );
+ if ( rp == 0 && re == 0 ) {
+ if ( ! StringEqual ( & pstr, & estr ) )
+ * notequal |= 8;
+ }
+ else if ( rc == 0 ) {
+ if ( rp != 0 )
+ rc = rp;
+ else
+ rc = re;
+ }
+
+ if ( VPathFromUri ( l ) != VPathFromUri ( r ) )
+ * notequal |= 0x10;
+
+ rp = VPathGetScheme_t ( l, & puri_type );
+ re = VPathGetScheme_t ( r, & euri_type );
+ if ( rp == 0 && re == 0 ) {
+ if ( memcmp ( & puri_type, & euri_type, sizeof euri_type) != 0 )
+ * notequal |= 0x20;
+ }
+ else if ( rc == 0 ) {
+ if ( rp != 0 )
+ rc = rp;
+ else
+ rc = re;
+ }
+
+ if ( VPathIsHighlyReliable ( l ) != VPathIsHighlyReliable ( r ) )
+ * notequal |= 0x40;
+
+ if ( l -> ext && r -> ext ) {
+ rp = VPathGetId ( l, & pstr );
+ re = VPathGetId ( r, & estr );
+ if ( rp == 0 && re == 0 ) {
+ if ( ! StringEqual ( & pstr, & estr ) )
+ * notequal |= 0x80;
+ }
+ else if ( rc == 0 ) {
+ if ( rp != 0 )
+ rc = rp;
+ else
+ rc = re;
+ }
+
+ rp = VPathGetTicket ( l, & pstr );
+ re = VPathGetTicket ( r, & estr );
+ if ( rp == 0 && re == 0 ) {
+ if ( ! StringEqual ( & pstr, & estr ) )
+ * notequal |= 0x100;
+ }
+ else if ( rc == 0 ) {
+ if ( rp != 0 )
+ rc = rp;
+ else
+ rc = re;
+ }
+
+ if ( l -> ext == eVPext && r -> ext == eVPext ) {
+ {
+ KTime_t tp = VPathGetModDate ( l );
+ KTime_t te = VPathGetModDate ( r );
+ if ( tp != te )
+ * notequal |= 0x200;
+ }
+ {
+ size_t p = VPathGetSize ( l );
+ size_t e = VPathGetSize ( r );
+ if ( p != e )
+ * notequal |= 0x400;
+ }
+ {
+ const uint8_t * p = VPathGetMd5 ( l );
+ const uint8_t * e = VPathGetMd5 ( r );
+ if ( ( p == NULL && e != NULL ) || ( e == NULL && p != NULL ) )
+ * notequal |= 0x800;
+ else if ( p != NULL ) {
+ int i = 0;
+ for ( i = 0; i < 16; ++i )
+ if ( p [ i ] != e [ i ] ) {
+ * notequal |= 0x1000;
+ break;
+ }
+ }
+ }
+ {
+ KTime_t tp = l -> expiration;
+ KTime_t te = r -> expiration;
+ if ( tp != te ) {
+/* TO IMPLEMENT KTime_t diff = abs ( l -> expiration - r -> expiration );*/
+ * notequal |= 0x2000;
+ }
+ }
+ }
+ }
+
+ return rc;
+}
+
+rc_t VPathEqual ( const VPath * l, const VPath * r, int * notequal ) {
+ return VPathClose ( l, r, notequal, 0 );
+}
diff --git a/libs/vfs/remote-services.c b/libs/vfs/remote-services.c
new file mode 100644
index 0000000..bf80844
--- /dev/null
+++ b/libs/vfs/remote-services.c
@@ -0,0 +1,3905 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * =============================================================================
+ *
+ */
+
+
+/******************************************************************************/
+
+
+#include <vfs/extern.h>
+
+#include <klib/container.h> /* BSTree */
+#include <klib/debug.h> /* DBGMSG */
+#include <klib/log.h> /* KLogLevel */
+#include <klib/out.h> /* KOutMsg */
+#include <klib/printf.h> /* string_printf */
+#include <klib/rc.h> /* RC */
+#include <klib/text.h> /* String */
+#include <klib/time.h> /* KTime */
+#include <klib/vector.h> /* Vector */
+
+#include <kfg/config.h> /* KConfigRelease */
+#include <kfg/kart-priv.h> /* KartItemMake2 */
+#include <kfg/repository.h> /* KRepositoryMgrRelease */
+
+#include <kns/http.h> /* KHttpRequest */
+#include <kns/http-priv.h> /* KClientHttpRequestFormatMsg */
+#include <kns/kns-mgr-priv.h> /* KNSManagerMakeReliableClientRequest */
+#include <kns/manager.h> /* KNSManager */
+#include <kns/stream.h> /* KStreamMakeFromBuffer */
+
+#include <kproc/timeout.h> /* TimeoutInit */
+
+#include <vfs/manager.h> /* VFSManager */
+#include <vfs/services.h> /* KServiceMake */
+
+#include "path-priv.h" /* VPathMakeFmt */
+#include "resolver-priv.h" /* VPathCheckFromNamesCGI */
+#include "services-priv.h"
+
+#include <ctype.h> /* isdigit */
+#include <string.h> /* memset */
+
+
+/******************************************************************************/
+
+
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
+
+
+/******************************************************************************/
+/* service type - names of search */
+typedef enum {
+ eSTnames,
+ eSTsearch,
+} EServiceType;
+
+
+/* request/response/processing helper objects */
+typedef struct {
+ struct KConfig * kfg;
+
+ const struct KNSManager * kMgr;
+ /* KNSManagerMakeReliableClientRequest */
+
+ const struct KRepositoryMgr * repoMgr;
+ /* KRepositoryMgrGetProtectedRepository */
+
+ uint32_t timeoutMs;
+} SHelper;
+
+
+/* raw string text */
+typedef struct { char * s; } SRaw;
+
+
+/******************************************************************************/
+/* SERVICE VERSIONS */
+#define VERSION_1_0 0x01000000
+#define VERSION_1_1 0x01010000
+#define VERSION_1_2 0x01020000
+#define VERSION_3_0 0x03000000
+/*#define VERSION_3_1 0x03010000
+#define VERSION_3_2 0x03020000*/
+
+
+/* version in server request / response */
+typedef struct {
+ SRaw raw;
+ ver_t version;
+ uint8_t major;
+ uint8_t minor;
+} SVersion;
+
+/* features that are different for different protocol versions ****************/
+static bool SVersionNotExtendedVPaths ( const SVersion * self ) {
+ assert ( self );
+ return self -> version == VERSION_1_0;
+}
+
+static bool SVersionBefore3_0 ( const SVersion * self ) {
+ assert ( self );
+ return self -> version < VERSION_3_0;
+}
+
+static bool SVersionHasRefseqCtx ( const SVersion * self ) {
+ assert ( self );
+ return self -> version < VERSION_3_0;
+}
+
+static bool SVersionAccInRequest ( const SVersion * self ) {
+ assert ( self );
+ return self -> version < VERSION_3_0;
+}
+
+static bool SVersionTypInRequest ( const SVersion * self ) {
+ assert ( self );
+ return self -> version == VERSION_3_0;
+}
+
+static bool SVersionHasMultpileObjects ( const SVersion * self ) {
+ assert ( self );
+ return self -> version >= VERSION_3_0;
+}
+
+static bool SVersionSingleUrl ( const SVersion * self ) {
+ assert ( self );
+ return self -> version < VERSION_3_0;
+}
+
+static bool SVersionResponseHasMultipeUrls ( const SVersion * self ) {
+ assert ( self );
+ return self -> version >= VERSION_3_0;
+}
+
+static bool SVersionResponseHasTimestamp ( const SVersion * self ) {
+ assert ( self );
+ return self -> version >= VERSION_3_0;
+}
+/******************************************************************************/
+
+
+/* server response header */
+typedef struct {
+ SRaw raw;
+ SVersion version;
+} SHeader;
+
+
+/* number of fields in different versions of name resolver protocol */
+#define N_NAMES1_0 5
+#define N_NAMES1_1 10
+#define N_NAMES3_0 15
+
+/* md5 checksum */
+typedef struct {
+ uint8_t md5 [ 16 ];
+ bool has_md5;
+} SMd5;
+
+
+/* response row parsed into named typed fields */
+typedef struct {
+ bool inited;
+ uint32_t ordId;
+ EObjectType objectType;
+ String accession; /* versios 1.1/1.2 only */
+ String objectId;
+ String name;
+ size_t size;
+ KTime_t date;
+ SMd5 md5;
+ String ticket;
+ String url;
+ String hUrl;
+ String fpUrl;
+ String hsUrl;
+ String flUrl;
+ String s3Url;
+ size_t vdbcacheSize;
+ KTime_t vdbcacheDate;
+ SMd5 vdbcacheMd5;
+ String vdbcacheUrl;
+ String hVdbcacheUrl;
+ String fpVdbcacheUrl;
+ String hsVdbcacheUrl;
+ String flVdbcacheUrl;
+ String s3VdbcacheUrl;
+ KTime_t expiration;
+ uint32_t code;
+ String message;
+} STyped;
+
+
+/* converter from server response string to a typed object */
+typedef rc_t TConverter ( void * dest, const String * src );
+typedef struct { TConverter * f; } SConverter;
+typedef void * TFieldGetter ( STyped * self, int n );
+
+typedef struct {
+ int n;
+ TFieldGetter * get;
+ TConverter ** f;
+} SConverters;
+
+
+/* response row parsed into array of Strings */
+typedef struct {
+ int n;
+ String s [ N_NAMES3_0 ];
+} SOrdered;
+
+
+/* server error row */
+struct KSrvError {
+ atomic32_t refcount;
+
+ rc_t rc;
+ uint32_t code;
+ String message;
+
+ String objectId;
+ EObjectType objectType;
+};
+
+
+/* EVPath is defined in ./path-priv.h */
+
+
+/* a response row */
+typedef struct {
+ SRaw raw;
+ SOrdered ordered;
+ STyped typed;
+ EVPath path;
+ VPathSet * set;
+} SRow;
+
+
+/* timestamp object */
+typedef struct {
+ SRaw raw;
+ KTime_t time;
+} STimestamp;
+
+
+/* server timestamp */
+typedef struct {
+ STimestamp server;
+ STimestamp local;
+} SServerTimestamp;
+
+
+/* server response = header; vector of response rows; generated objects */
+typedef struct {
+ EServiceType serviceType;
+ SHeader header;
+ Vector rows;
+ KSrvResponse * list;
+ Kart * kart;
+ SServerTimestamp timestamp;
+} SResponse;
+
+
+/* key-value */
+typedef struct {
+ String k;
+ String v;
+} SKV;
+
+
+/* helper object to add cgi parameters to KHttpRequest */
+typedef struct {
+ KHttpRequest * httpReq;
+ rc_t rc;
+} SHttpRequestHelper;
+
+
+/* cgi request info */
+typedef struct {
+ bool inited;
+ char * cgi;
+ Vector params;
+} SCgiRequest;
+
+
+/* request object */
+typedef struct {
+ char * objectId;
+ EObjectType objectType;
+ int ordId;
+} SObject;
+
+
+/* service request data ( objects to query ) */
+typedef struct {
+ SObject object [ 256 ];
+ uint32_t objects;
+ bool refseq_ctx;
+} SRequestData;
+
+
+typedef struct {
+ BSTNode n;
+ char * ticket;
+ uint32_t project;
+} BSTItem;
+
+
+/* tickets - access authorization keys */
+typedef struct {
+ BSTree ticketsToProjects;
+ Vector tickets;
+ KDataBuffer str;
+ size_t size;
+ rc_t rc;
+} STickets;
+
+
+/* service request */
+typedef struct {
+ EServiceType serviceType;
+ SVersion version;
+ SCgiRequest cgiReq;
+ SRequestData request;
+ STickets tickets;
+ int errorsToIgnore;
+ VRemoteProtocols protocols;
+} SRequest;
+
+
+/* service object */
+struct KService {
+ SHelper helper;
+ SRequest req;
+ SResponse resp;
+};
+
+
+/******************************************************************************/
+
+
+/* SHelper ********************************************************************/
+static rc_t SHelperInit ( SHelper * self, const KNSManager * kMgr ) {
+ rc_t rc = 0;
+ assert ( self );
+ memset ( self, 0, sizeof * self );
+ if ( kMgr == NULL ) {
+ KNSManager * kns = NULL;
+ rc = KNSManagerMake ( & kns );
+ kMgr = kns;
+ }
+ else {
+ rc = KNSManagerAddRef ( kMgr );
+ }
+ if ( rc == 0) {
+ self -> kMgr = kMgr;
+ }
+ self -> timeoutMs = 5000;
+ return rc;
+}
+
+
+static rc_t SHelperFini ( SHelper * self) {
+ rc_t rc = 0;
+
+ assert ( self );
+
+ RELEASE ( KConfig , self -> kfg );
+ RELEASE ( KNSManager , self -> kMgr );
+ RELEASE ( KRepositoryMgr, self -> repoMgr );
+
+ memset ( self, 0, sizeof * self );
+
+ return rc;
+}
+
+
+static rc_t SHelperInitKfg ( SHelper * self ) {
+ rc_t rc = 0;
+
+ assert ( self );
+
+ if ( self -> kfg == NULL )
+ rc = KConfigMake ( & self -> kfg, NULL );
+
+ return rc;
+}
+
+
+static rc_t SHelperInitRepoMgr ( SHelper * self ) {
+ rc_t rc = 0;
+
+ assert ( self );
+
+ if ( self -> repoMgr == NULL ) {
+ rc = SHelperInitKfg ( self );
+ if ( rc != 0 )
+ return rc;
+
+ rc = KConfigMakeRepositoryMgrRead ( self -> kfg, & self -> repoMgr );
+ }
+
+ return rc;
+}
+
+
+/* get from kfg, otherwise use hardcoded */
+static VRemoteProtocols SHelperDefaultProtocols ( SHelper * self ) {
+ VRemoteProtocols protocols = DEFAULT_PROTOCOLS;
+
+ assert ( self );
+
+ SHelperInitKfg ( self );
+
+ KConfigReadRemoteProtocols ( self -> kfg, & protocols );
+
+ return protocols;
+}
+
+
+/* try to get cgi from kfg, otherwise use hardcoded */
+static
+rc_t SHelperResolverCgi ( SHelper * self, bool aProtected,
+ char * buffer, size_t bsize )
+{
+ const char man [] = "/repository/remote/main/CGI/resolver-cgi";
+ const char prt [] = "/repository/remote/protected/CGI/resolver-cgi";
+ const char cgi[]= "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi";
+ rc_t rc = 0;
+ const char * path = aProtected ? prt : man;
+ assert ( self );
+ rc = SHelperInitKfg ( self );
+ if ( rc == 0 ) {
+ rc = KConfigRead ( self -> kfg, path, 0, buffer, bsize, NULL, NULL );
+ if ( rc != 0 ) {
+ if ( buffer == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+ if ( bsize < sizeof cgi )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcBuffer,
+ rcInsufficient );
+ string_copy ( buffer, bsize, cgi, sizeof cgi );
+ }
+ }
+ return rc;
+}
+
+
+static
+rc_t SHelperProjectToTicket ( SHelper * self, uint32_t projectId,
+ char * buffer, size_t bsize, size_t * ticket_size )
+{
+ rc_t rc = 0;
+
+ const KRepository * repo = NULL;
+
+ assert ( self );
+
+ rc = SHelperInitRepoMgr ( self );
+
+ rc = KRepositoryMgrGetProtectedRepository ( self -> repoMgr, projectId,
+ & repo );
+ if ( rc != 0 )
+ return rc;
+
+ rc = KRepositoryDownloadTicket ( repo, buffer, bsize, ticket_size );
+
+ RELEASE ( KRepository, repo );
+
+ return rc;
+}
+
+
+/* SRaw ***********************************************************************/
+static void SRawInit ( SRaw * self, char * s ) {
+ assert ( self );
+
+ self -> s = s;
+}
+
+
+static rc_t SRawAlloc ( SRaw * self, const char * s, size_t sz ) {
+ char * p = NULL;
+
+ if ( sz == 0 )
+ p = string_dup_measure ( s, NULL );
+ else
+ p = string_dup ( s, sz );
+
+ if ( p == NULL )
+ return RC ( rcVFS, rcPath, rcAllocating, rcMemory, rcExhausted );
+
+ SRawInit ( self, p );
+
+ return 0;
+}
+
+
+static rc_t SRawFini ( SRaw * self ) {
+ if ( self != NULL ) {
+ free ( self -> s );
+ self -> s = NULL;
+ }
+
+ return 0;
+}
+
+
+/* SVersion *******************************************************************/
+static rc_t SVersionInit
+ ( SVersion * self, const char * src, EServiceType serviceType )
+{
+ const char * s = src;
+
+ assert ( self );
+
+ memset ( self, 0, sizeof * self );
+
+ if ( s == NULL ) {
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMessage, rcBadVersion );
+ }
+
+ if ( * s != '#' ) {
+ if ( serviceType != eSTnames || * s == '\0' || ! isdigit ( * s ) )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMessage, rcBadVersion );
+ }
+ else
+ ++ s;
+
+ if ( serviceType == eSTsearch ) {
+ const char version [] = "version ";
+ size_t sz = sizeof version - 1;
+ if ( string_cmp ( s, sz, version, sz, ( uint32_t ) sz ) != 0 )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMessage, rcBadVersion );
+ s += sz;
+ }
+
+ if ( * s == '\0' ) {
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMessage, rcBadVersion );
+ }
+
+ {
+ char * end = NULL;
+ uint64_t l = strtoul ( s, & end, 10 );
+ if ( end == NULL || * end != '.' ) {
+ return RC ( rcVFS, rcQuery, rcResolving, rcMessage, rcCorrupt );
+ }
+ self -> major = l;
+ s = ++ end;
+
+ l = strtoul ( s, & end, 10 );
+ if ( end == NULL || * end != '\0' ) {
+ return RC ( rcVFS, rcQuery, rcResolving, rcMessage, rcCorrupt );
+ }
+ self -> minor = l;
+
+ self -> version = self -> major << 24 | self -> minor << 16;
+
+ return SRawAlloc ( & self -> raw, src, 0 );
+ }
+}
+
+
+static rc_t SVersionFini ( SVersion * self ) {
+ rc_t rc = 0;
+ assert ( self );
+ rc = SRawFini ( & self -> raw );
+ memset ( self, 0, sizeof * self );
+ return rc;
+}
+
+
+static rc_t SVersionToString ( const SVersion * self, char ** s ) {
+ size_t num_writ = 0;
+ char tmp [ 1 ];
+ assert ( self && s );
+ string_printf ( tmp, 1, & num_writ, "%u.%u", self -> major, self -> minor );
+ ++ num_writ;
+ * s = ( char * ) malloc ( num_writ );
+ if ( * s == NULL ) {
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ }
+ return string_printf ( * s, num_writ,
+ & num_writ, "%u.%u", self -> major, self -> minor );
+}
+
+
+/* SHeader ********************************************************************/
+static rc_t SHeaderMake
+ ( SHeader * self, const String * src, EServiceType serviceType )
+{
+ rc_t rc = 0;
+
+ assert ( self && src );
+
+ memset ( self, 0, sizeof * self );
+
+ rc = SRawAlloc ( & self -> raw, src -> addr, src -> size );
+
+ if ( rc == 0 )
+ rc = SVersionInit ( & self -> version, self -> raw . s, serviceType );
+
+ return rc;
+}
+
+
+static rc_t SHeaderFini ( SHeader * self ) {
+ rc_t rc = 0;
+ if ( self != NULL ) {
+ rc_t r2 = SRawFini ( & self -> raw );
+ rc = SVersionFini ( & self -> version );
+ if ( rc == 0 ) {
+ rc = r2;
+ }
+ }
+ return rc;
+}
+
+
+/* SProtocol ******************************************************************/
+typedef struct {
+ const char * text;
+ VRemoteProtocols protocol;
+} SProtocol;
+
+
+static VRemoteProtocols SProtocolGet ( const String * url ) {
+ size_t i = 0;
+ SProtocol protocols [] = {
+ { "http:", eProtocolHttp },
+ { "fasp:", eProtocolFasp },
+ { "https:",eProtocolHttps},
+ { "file:", eProtocolFile },
+ { "s3:" , eProtocolS3 },
+ };
+ if ( url == NULL || url -> addr == NULL || url -> size == 0 ) {
+ return eProtocolNone;
+ }
+ for ( i = 0; i < sizeof protocols / sizeof protocols [ 0 ]; ++ i ) {
+ uint32_t sz = string_measure ( protocols [ i ] . text, NULL );
+ if ( string_cmp ( url -> addr, sz, protocols [ i ] . text, sz,
+ ( uint32_t ) sz ) == 0 )
+ {
+ return protocols [ i ] . protocol;
+ }
+ }
+ return eProtocolNone;
+}
+
+
+/* STyped *********************************************************************/
+static rc_t STypedInitUrls ( STyped * self ) {
+ VRemoteProtocols protocol = eProtocolNone;
+ String * str = NULL;
+ String * dst = NULL;
+ assert ( self );
+ str = & self -> url;
+ while ( str -> size > 0 ) {
+ size_t len = 0;
+ char * n = string_chr ( str -> addr, str -> size, '$' );
+ if ( n != NULL ) {
+ len = n - str -> addr;
+ }
+ else {
+ len = str -> size;
+ }
+ protocol = SProtocolGet ( str );
+ switch ( protocol ) {
+ case eProtocolFasp:
+ dst = & self -> fpUrl;
+ break;
+ case eProtocolFile:
+ dst = & self -> flUrl;
+ break;
+ case eProtocolHttp:
+ dst = & self -> hUrl;
+ break;
+ case eProtocolHttps:
+ dst = & self -> hsUrl;
+ break;
+ case eProtocolS3:
+ dst = & self -> s3Url;
+ break;
+ default:
+ return RC ( rcVFS, rcQuery, rcResolving, rcMessage, rcCorrupt );
+ }
+ StringInit ( dst, str -> addr, len, len );
+ if ( n != NULL )
+ ++ len;
+ str -> addr += len;
+ if ( str -> size >= len )
+ str -> size -= len;
+ else
+ str -> size = 0;
+ }
+ str = & self -> vdbcacheUrl;
+ while ( str -> size > 0 ) {
+ size_t len = 0;
+ char * n = string_chr ( str -> addr, str -> size, '$' );
+ if ( n != NULL ) {
+ len = n - str -> addr;
+ }
+ else {
+ len = str -> size;
+ }
+ protocol = SProtocolGet ( str );
+ switch ( protocol ) {
+ case eProtocolFasp:
+ dst = & self -> fpVdbcacheUrl;
+ break;
+ case eProtocolFile:
+ dst = & self -> flVdbcacheUrl;
+ break;
+ case eProtocolHttp:
+ dst = & self -> hVdbcacheUrl;
+ break;
+ case eProtocolHttps:
+ dst = & self -> hsVdbcacheUrl;
+ break;
+ case eProtocolS3:
+ dst = & self -> s3VdbcacheUrl;
+ break;
+ default:
+ return RC ( rcVFS, rcQuery, rcResolving, rcMessage, rcCorrupt );
+ }
+ StringInit ( dst, str -> addr, len, len );
+ if ( n != NULL )
+ ++ len;
+ str -> addr += len;
+ if ( str -> size >= len )
+ str -> size -= len;
+ else
+ str -> size = 0;
+ }
+ return 0;
+}
+
+
+static
+rc_t STypedInit ( STyped * self, const SOrdered * raw, const SConverters * how,
+ const SVersion * version )
+{
+ rc_t rc = 0;
+ int i = 0;
+ assert ( self && raw && how && version );
+ memset ( self, 0, sizeof * self );
+
+ if ( raw -> n != how -> n ) /* BREAK */
+ return RC ( rcVFS, rcQuery, rcResolving, rcName, rcUnexpected );
+
+ for ( i = 0; i < raw -> n; ++i ) {
+ void * dest = how -> get ( self, i );
+ if ( dest == NULL ) {
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcMessage, rcCorrupt );
+ break;
+ }
+ {
+ TConverter * f = how -> f [ i ];
+ if ( f == NULL ) {
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcFunction, rcNotFound );
+ break;
+ }
+ rc = f ( dest, & raw -> s [ i ] );
+ if ( rc != 0 )
+ break; /* BREAK */
+ }
+ }
+
+ if ( rc == 0 )
+ if ( SVersionResponseHasMultipeUrls ( version ) )
+ rc = STypedInitUrls ( self );
+
+ if ( rc == 0 )
+ self -> inited = true;
+
+ return rc;
+}
+
+
+/* CONVERTERS */
+/* functions to initialize objects from response row field Strings ************/
+static
+bool cmpStringAndObjectType ( const String * s, const char * val )
+{
+ size_t sz = string_size (val );
+ String v;
+ StringInit ( & v, val, sz, sz );
+ return StringCompare ( s, & v ) == 0;
+}
+
+
+/* N.B. DO NOT FREE RETURNED STRING !!! */
+static const char * ObjectTypeToString ( EObjectType self ) {
+ switch ( self ) {
+ case eOT_undefined : return "";
+ case eOT_dbgap : return "dbgap";
+ case eOT_provisional : return "provisional";
+ case eOT_srapub : return "srapub";
+ case eOT_sragap : return "sragap";
+ case eOT_srapub_source:return "srapub_source";
+ case eOT_sragap_source:return "sragap_source";
+ case eOT_srapub_files: return "srapub_files";
+ case eOT_sragap_files: return "sragap_files";
+ case eOT_refseq : return "refseq";
+ case eOT_wgs : return "wgs";
+ case eOT_na : return "na";
+ case eOT_nakmer : return "nakmer";
+ default: assert ( 0 ); return "";
+ }
+}
+
+
+static EObjectType StringToObjectType ( const String * s ) {
+ if ( cmpStringAndObjectType ( s, "" ) ) {
+ return eOT_empty;
+ }
+ if ( cmpStringAndObjectType ( s, "dbgap" ) ) {
+ return eOT_dbgap;
+ }
+ if ( cmpStringAndObjectType ( s, "provisional" ) ) {
+ return eOT_provisional;
+ }
+ if ( cmpStringAndObjectType ( s, "srapub" ) ) {
+ return eOT_srapub;
+ }
+ if ( cmpStringAndObjectType ( s, "sragap" ) ) {
+ return eOT_sragap;
+ }
+ if ( cmpStringAndObjectType ( s, "srapub_source" ) ) {
+ return eOT_srapub_source;
+ }
+ if ( cmpStringAndObjectType ( s, "srapub_files" ) ) {
+ return eOT_srapub_files;
+ }
+ if ( cmpStringAndObjectType ( s, "sragap_files") ) {
+ return eOT_sragap_files;
+ }
+ if ( cmpStringAndObjectType ( s, "refseq") ) {
+ return eOT_refseq;
+ }
+ if ( cmpStringAndObjectType ( s, "wgs") ) {
+ return eOT_wgs;
+ }
+ if ( cmpStringAndObjectType ( s, "na") ) {
+ return eOT_na;
+ }
+ if ( cmpStringAndObjectType ( s, "nakmer") ) {
+ return eOT_nakmer;
+ }
+ return eOT_undefined;
+}
+
+
+static rc_t EObjectTypeInit ( void * p, const String * src ) {
+ EObjectType * self = ( EObjectType * ) p;
+ EObjectType t = StringToObjectType ( src );
+ if ( t == eOT_undefined ) {
+ return RC ( rcVFS, rcQuery, rcExecuting, rcType, rcIncorrect );
+ }
+ assert ( self );
+ * self = t;
+ return 0;
+}
+
+
+static rc_t aStringInit ( void * p, const String * src ) {
+ String * self = ( String * ) p;
+ assert ( src );
+ StringInit ( self, src -> addr, src -> size, src -> len );
+ return 0;
+}
+
+
+static rc_t size_tInit ( void * p, const String * src ) {
+ rc_t rc = 0;
+ size_t * self = ( size_t * ) p;
+ size_t s = 0;
+ if ( src -> size != 0 && src -> len != 0 ) {
+ s = StringToU64 ( src, & rc );
+ }
+ if ( rc == 0 ) {
+ assert ( self );
+ * self = s;
+ }
+ return rc;
+}
+
+
+static rc_t uint32_tInit ( void * p, const String * src ) {
+ rc_t rc = 0;
+ uint32_t * self = ( uint32_t * ) p;
+ uint32_t s = StringToU64 ( src, & rc );
+ if ( rc == 0 ) {
+ assert ( self );
+ * self = s;
+ }
+ return rc;
+}
+
+
+#if 0 && LINUX
+#define TODO 1;
+static
+rc_t YYYY_MM_DDThh_mm_ssZToTm ( const char * src, struct tm * result )
+{
+/*YYYY-MM-DDThh:mm:ssTZD ISO 8601
+tm_sec int seconds after the minute 0-61*
+tm_min int minutes after the hour 0-59
+tm_hour int hours since midnight 0-23
+tm_mday int day of the month 1-31
+tm_mon int months since January 0-11
+tm_year int years since 1900
+tm_wday int days since Sunday 0-6
+tm_yday int days since January 1 0-365
+tm_isdst int Daylight Saving Time flag */
+ int i = 0;
+ int tmp = 0;
+ char c = 0;
+ assert ( src && result );
+ memset ( result, 0, sizeof * result );
+ for ( i = 0, tmp = 0; i < 4; ++ i ) {
+ char c = src [ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = tmp * 10 + c - '0';
+ }
+ if ( tmp < 1900 )
+ return TODO;
+ result -> tm_year = tmp - 1900;
+ if ( src [ i ] != '-' )
+ return TODO;
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = c - '0';
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = tmp * 10 + c - '0';
+ if ( tmp == 0 || tmp > 12 )
+ return TODO;
+ result -> tm_mon = tmp - 1;
+ c = src [ ++ i ];
+ if ( c != '-' )
+ return TODO;
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = c - '0';
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = tmp * 10 + c - '0';
+ if ( tmp == 0 || tmp > 31 )
+ return TODO;
+ result -> tm_mday = tmp;
+ c = src [ ++ i ];
+ if ( c != 'T' )
+ return TODO;
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = c - '0';
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = tmp * 10 + c - '0';
+ if ( tmp > 23 )
+ return TODO;
+ result -> tm_hour = tmp;
+ c = src [ ++ i ];
+ if ( c != ':' )
+ return TODO;
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = c - '0';
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = tmp * 10 + c - '0';
+ if ( tmp > 59 )
+ return TODO;
+ result -> tm_min = tmp;
+ c = src [ ++ i ];
+ if ( c != ':' )
+ return TODO;
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = c - '0';
+ c = src [ ++ i ];
+ if ( ! isdigit ( c ) )
+ return TODO;
+ tmp = tmp * 10 + c - '0';
+ if ( tmp > 59 )
+ return TODO;
+ result -> tm_sec = tmp;
+ c = src [ ++ i ];
+ if ( c != 'Z' )
+ return TODO;
+ /*time_t time = 0;
+ struct tm * t = gmtime_r ( & time, result );*/
+ return 0;
+}
+#endif
+
+
+static rc_t KTimeInit ( void * p, const String * src ) {
+ rc_t rc = 0;
+
+ KTime_t * self = ( KTime_t * ) p;
+
+ assert ( self && src );
+
+ if ( src -> addr != NULL && src -> size > 0 )
+ * self = StringToU64 ( src, & rc );
+
+ return rc;
+}
+
+
+static rc_t KTimeInitFromIso8601 ( void * p, const String * src ) {
+ rc_t rc = 0;
+
+ KTime_t * self = ( KTime_t * ) p;
+
+ assert ( self && src );
+
+ if ( src -> addr != NULL && src -> size > 0 ) {
+ KTime kt;
+ const KTime * t = KTimeFromIso8601 ( & kt, src -> addr, src -> size );
+ if ( t == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcItem, rcIncorrect );
+ else
+ * self = KTimeMakeTime ( & kt );
+ }
+
+ return rc;
+}
+
+
+static int getDigit ( char c, rc_t * rc ) {
+ assert ( rc );
+
+ if ( * rc != 0 )
+ return 0;
+
+ c = tolower ( c );
+ if ( ! isdigit ( c ) && c < 'a' && c > 'f' ) {
+ * rc = RC ( rcVFS, rcQuery, rcExecuting, rcItem, rcIncorrect );
+ return 0;
+ }
+
+ if ( isdigit ( c ) )
+ return c - '0';
+
+ return c - 'a' + 10;
+}
+
+
+static rc_t md5Init ( void * p, const String * src ) {
+ SMd5 * md5 = p;
+
+ assert ( src && src -> addr && md5 );
+
+ md5 -> has_md5 = false;
+
+ switch ( src -> size ) {
+ case 0:
+ return 0;
+ case 32: {
+ rc_t rc = 0;
+ int i = 0;
+ for ( i = 0; i < 16 && rc == 0; ++ i ) {
+ md5 -> md5 [ i ] = getDigit ( src -> addr [ 2 * i ], & rc )
+ * 16;
+ md5 -> md5 [ i ] += getDigit ( src -> addr [ 2 * i + 1 ],
+ & rc );
+ }
+ md5 -> has_md5 = rc == 0;
+ return rc;
+ }
+ default:
+ return RC ( rcVFS, rcQuery, rcExecuting, rcItem, rcIncorrect );
+ }
+}
+
+
+/* SConverters ****************************************************************/
+/* converter from names-1.0 response row to STyped object */
+static void * STypedGetFieldNames1_0 ( STyped * self, int n ) {
+ assert ( self );
+ switch ( n ) {
+ case 0: return & self -> accession;
+ case 1: return & self -> ticket;
+ case 2: return & self -> hUrl;
+ case 3: return & self -> code;
+ case 4: return & self -> message;
+ }
+ return 0;
+}
+
+
+static const SConverters * SConvertersNames1_0Make ( void ) {
+ static TConverter * f [ N_NAMES1_0 + 1 ] = {
+ aStringInit,
+ aStringInit,
+ aStringInit,
+ uint32_tInit,
+ aStringInit, NULL };
+ static const SConverters c = {
+ N_NAMES1_0,
+ STypedGetFieldNames1_0, f };
+ return & c;
+}
+
+
+/* converter from names-1.1 response row to STyped object */
+static void * STypedGetFieldNames1_1 ( STyped * self, int n ) {
+ assert ( self);
+ switch ( n ) {
+ case 0: return & self -> accession;
+ case 1: return & self -> objectId;
+ case 2: return & self -> name;
+ case 3: return & self -> size;
+ case 4: return & self -> date;
+ case 5: return & self -> md5;
+ case 6: return & self -> ticket;
+ case 7: return & self -> hUrl;
+ case 8: return & self -> code;
+ case 9: return & self -> message;
+ }
+ return 0;
+}
+
+
+static const SConverters * SConvertersNames1_1Make ( void ) {
+ static TConverter * f [ N_NAMES1_1 + 1 ] = {
+ aStringInit,
+ aStringInit,
+ aStringInit,
+ size_tInit,
+ KTimeInitFromIso8601,
+ md5Init,
+ aStringInit,
+ aStringInit,
+ uint32_tInit,
+ aStringInit, NULL };
+ static const SConverters c = {
+ N_NAMES1_1,
+ STypedGetFieldNames1_1, f };
+ return & c;
+}
+
+
+static const SConverters * SConvertersNames1_2Make ( void ) {
+ static TConverter * f [ N_NAMES1_1 + 1 ] = {
+ aStringInit,
+ aStringInit,
+ aStringInit,
+ size_tInit,
+ KTimeInitFromIso8601,
+ md5Init,
+ aStringInit,
+ aStringInit,
+ uint32_tInit,
+ aStringInit, NULL };
+ static const SConverters c = {
+ N_NAMES1_1,
+ STypedGetFieldNames1_1, f };
+ return & c;
+}
+
+
+/* converter from names-3.0 response row to STyped object */
+static void * STypedGetFieldNames3_0 ( STyped * self, int n ) {
+ assert ( self);
+ switch ( n ) {
+ case 0: return & self -> ordId;
+ case 1: return & self -> objectType;
+ case 2: return & self -> objectId;
+ case 3: return & self -> size;
+ case 4: return & self -> date;
+ case 5: return & self -> md5;
+ case 6: return & self -> ticket;
+ case 7: return & self -> url;
+ case 8: return & self -> vdbcacheSize;
+ case 9: return & self -> vdbcacheDate;
+ case 10: return & self -> vdbcacheMd5;
+ case 11: return & self -> vdbcacheUrl;
+ case 12: return & self -> expiration;
+ case 13: return & self -> code;
+ case 14: return & self -> message;
+ }
+ return 0;
+}
+
+
+static const SConverters * SConvertersNames3_0Make ( void ) {
+ static TConverter * f [ N_NAMES3_0 + 1 ] = {
+ uint32_tInit, /* 0 ord-id */
+ EObjectTypeInit, /* 1 object-type */
+ aStringInit, /* 2 object-id */
+ size_tInit, /* 3 size */
+ KTimeInitFromIso8601,/* 4 date */
+ md5Init, /* 5 md5 */
+ aStringInit, /* 6 ticket */
+ aStringInit, /* 7 url */
+ size_tInit, /* 8 vdbcache-size */
+ KTimeInitFromIso8601,/* 9 vdbcache-date */
+ md5Init, /* 10 vdbcache-md5 */
+ aStringInit, /* 11 vdbcache-url */
+ KTimeInit, /* 12 expiration */
+ uint32_tInit, /* 13 status-code */
+ aStringInit, /* 14 message */
+ NULL };
+ static const SConverters c = {
+ N_NAMES3_0,
+ STypedGetFieldNames3_0, f };
+ return & c;
+}
+
+
+/* converter factory function */
+static
+rc_t SConvertersMake ( const SConverters ** self, SHeader * header )
+{
+ assert ( self && header );
+ switch ( header -> version. version ) {
+ case VERSION_1_0:
+ * self = SConvertersNames1_0Make ();
+ return 0;
+ case VERSION_1_1:
+ * self = SConvertersNames1_1Make ();
+ return 0;
+ case VERSION_1_2:
+ * self = SConvertersNames1_2Make ();
+ return 0;
+ case VERSION_3_0:
+ * self = SConvertersNames3_0Make ();
+ return 0;
+ default:
+ * self = NULL;
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMessage, rcBadVersion );
+ }
+}
+
+
+/* SOrdered *******************************************************************/
+static
+rc_t SOrderedInit ( SOrdered * self, const SRaw * raw, int fields )
+{
+ assert ( self && raw );
+ memset ( self, 0, sizeof * self );
+ {
+ const char * str = raw -> s;
+ size_t size = string_size ( str );
+ while ( size > 0 ) {
+ size_t len = 0;
+ char * n = string_chr ( str, size, '|' );
+ if ( n != NULL )
+ len = n - str;
+ else
+ len = size;
+ if ( self -> n >= fields ) {
+ return RC ( rcVFS, rcQuery, rcResolving, rcName, rcExcessive );
+ }
+ else {
+ String * s = & self -> s [ self -> n ++ ];
+ StringInit ( s, str, len, len );
+ if ( n != NULL )
+ ++ len;
+ str += len;
+ if ( size >= len )
+ size -= len;
+ else
+ size = 0;
+ }
+ }
+ }
+ return 0;
+}
+
+
+/* KSrvError ******************************************************************/
+static
+rc_t KSrvErrorMake ( const KSrvError ** self,
+ const STyped * src, rc_t aRc )
+{
+ KSrvError * o = NULL;
+ assert ( self && aRc );
+ o = ( KSrvError * ) calloc ( 1, sizeof * o );
+ if ( o == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+
+ if ( src != NULL ) {
+ o -> message . addr = string_dup ( src -> message . addr,
+ src -> message . size );
+ if ( o -> message . addr == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ o -> message . size = src -> message . size;
+ o -> message . len = src -> message . len;
+
+ o -> objectId . addr = string_dup ( src -> objectId . addr,
+ src -> objectId . size );
+ if ( o -> objectId . addr == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ o -> objectId . size = src -> objectId . size;
+ o -> objectId . len = src -> objectId . len;
+
+ o -> objectType = src -> objectType;
+
+ o -> code = src -> code;
+ }
+
+ o -> rc = aRc;
+
+ atomic32_set ( & o -> refcount, 1 );
+
+ * self = o;
+
+ return 0;
+}
+
+
+rc_t KSrvErrorAddRef ( const KSrvError * cself ) {
+ KSrvError * self = ( KSrvError * ) cself;
+
+ if ( self != NULL )
+ atomic32_inc ( & ( ( KSrvError * ) self ) -> refcount );
+
+ return 0;
+}
+
+
+rc_t KSrvErrorRelease ( const KSrvError * cself ) {
+ rc_t rc = 0;
+
+ KSrvError * self = ( KSrvError * ) cself;
+
+ if ( self != NULL && atomic32_dec_and_test ( & self -> refcount ) ) {
+ free ( ( void * ) self -> message . addr );
+ free ( ( void * ) self -> objectId . addr );
+ memset ( self, 0, sizeof * self );
+ free ( ( void * ) self );
+ }
+
+ return rc;
+}
+
+
+/* Rc - rc code corresponding to this Error */
+rc_t KSrvErrorRc ( const KSrvError * self, rc_t * rc ) {
+ rc_t dummy = 0;
+ if ( rc == NULL )
+ rc = & dummy;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ * rc = self -> rc;
+
+ return 0;
+}
+
+
+/* Code - Status-Code returned by server */
+rc_t KSrvErrorCode ( const KSrvError * self, uint32_t * code ) {
+ uint32_t dummy = 0;
+ if ( code == NULL )
+ code = & dummy;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ * code = self -> code;
+
+ return 0;
+}
+
+
+/* returns pointers to internal String data
+ * Strings remain valid while "self" is valid */
+/* Message - message returned by server */
+rc_t KSrvErrorMessage ( const KSrvError * self, String * message ) {
+ String dummy;
+ if ( message == NULL )
+ message = & dummy;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ * message = self -> message;
+
+ return 0;
+}
+
+
+/* Object - Object-Id/Object-Type that produced this Error */
+rc_t KSrvErrorObject ( const KSrvError * self,
+ String * id, EObjectType * type )
+{
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ if ( id != NULL )
+ * id = self -> objectId;
+
+ if ( type != NULL )
+ * type = self -> objectType;
+
+ return 0;
+}
+
+
+/* EVPath *********************************************************************/
+static bool VPathMakeOrNot ( VPath ** new_path, const String * src,
+ const String * ticket, const STyped * typed, bool ext, rc_t * rc,
+ bool useDates )
+{
+ String bug;
+ memset ( & bug, 0, sizeof bug );
+
+ assert ( new_path && src && typed && rc );
+
+ if ( * rc != 0 || src -> len == 0 )
+ return false;
+ else {
+ const String * id = & typed -> objectId;
+ if ( id -> size == 0 )
+ id = & typed -> accession;
+
+ if ( id -> size == 0 ) {
+/* compensate current names.cgi-v3.0 bug: it does not return id for object-id-s
+ */
+ if ( src -> size > 0 &&
+ isdigit ( src -> addr [ src -> size - 1 ] ) )
+ {
+ size_t s = 2;
+ bug . addr = src -> addr + src -> size - 1;
+ bug . size = 1;
+ for ( s = 2; s <= src -> size
+ && isdigit ( src -> addr [ src -> size - s ] );
+ ++ s )
+ {
+ -- bug . addr;
+ ++ bug . size;
+ }
+ bug . len = bug . size;
+ id = & bug;
+ }
+ }
+
+ if ( src -> size == 0 )
+ assert ( src -> addr != NULL );
+
+ * rc = VPathMakeFromUrl ( new_path, src, ticket, ext, id, typed -> size,
+ useDates ? typed -> date : 0,
+ typed -> md5 . has_md5 ? typed -> md5 . md5 : NULL,
+ useDates ? typed -> expiration : 0 );
+ if ( * rc == 0 )
+ VPathMarkHighReliability ( * new_path, true );
+
+ return true;
+ }
+}
+
+
+static rc_t EVPathInitMapping
+ ( EVPath * self, const STyped * src, const SVersion * version )
+{
+ rc_t rc = 0;
+ const VPath * vsrc = NULL;
+ assert ( self && src && version );
+ if ( self -> http == NULL && self -> fasp == NULL ) {
+ return 0;
+ }
+ vsrc = self -> http ? self -> http : self -> fasp;
+ rc = VPathCheckFromNamesCGI ( vsrc, & src -> ticket,
+ ( const struct VPath ** ) ( & self -> mapping ) );
+ if ( rc == 0) {
+ if ( SVersionBefore3_0 ( version ) ) {
+ if ( src -> ticket . size != 0 ) {
+ if ( src -> accession . size != 0 )
+ rc = VPathMakeFmt ( & self -> mapping, "ncbi-acc:%S?tic=%S",
+ & src -> accession, & src -> ticket );
+ else if ( src -> name . size == 0 )
+ return 0;
+ else
+ rc = VPathMakeFmt ( & self -> mapping,
+ "ncbi-file:%S?tic=%S", & src -> name, & src -> ticket );
+ }
+ else if ( src -> accession . size != 0 )
+ rc = VPathMakeFmt
+ ( & self -> mapping, "ncbi-acc:%S", & src -> accession );
+ else if ( src -> name . size == 0 )
+ return 0;
+ else
+ rc = VPathMakeFmt
+ ( & self -> mapping, "ncbi-file:%S", & src -> name );
+ }
+ else {
+ if ( src -> ticket . size != 0 ) {
+ if ( src -> objectId . size != 0 &&
+ src -> objectType == eOT_sragap )
+ {
+ rc = VPathMakeFmt ( & self -> mapping, "ncbi-acc:%S?tic=%S",
+ & src -> objectId, & src -> ticket );
+ }
+ else {
+ if ( src -> objectId . size == 0) {
+ return 0;
+ }
+ else {
+ rc = VPathMakeFmt ( & self -> mapping,
+ "ncbi-file:%S?tic=%S",
+ & src -> objectId, & src -> ticket );
+ }
+ }
+ }
+ else
+ if ( src -> objectId . size != 0 &&
+ src -> objectType == eOT_sragap )
+ {
+ rc = VPathMakeFmt
+ ( & self -> mapping, "ncbi-acc:%S", & src -> objectId );
+ }
+ else {
+ if ( src -> objectId . size == 0 )
+ return 0;
+ else
+ rc = VPathMakeFmt (
+ & self -> mapping, "ncbi-file:%S", & src -> objectId );
+ }
+ }
+
+ if ( rc == 0 )
+ return 0;
+
+ RELEASE ( VPath, self -> http );
+ RELEASE ( VPath, self -> fasp );
+ }
+
+ return rc;
+}
+
+
+static rc_t EVPathInit ( EVPath * self, const STyped * src,
+ const SRequest * req, rc_t * r, const char * acc )
+{
+ rc_t rc = 0;
+ bool made = false;
+ KLogLevel lvl = klogInt;
+ assert ( self && src && r );
+
+ switch ( src -> code / 100 ) {
+ case 0:
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcMessage, rcCorrupt );
+ break;
+
+ case 1:
+ /* informational response
+ not much we can do here */
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcError, rcUnexpected );
+ break;
+
+ case 2:
+ /* successful response
+ but can only handle 200 */
+ if ( src -> code == 200 ) {
+ bool ext = true;
+ assert ( req );
+ if ( req -> serviceType == eSTnames &&
+ SVersionNotExtendedVPaths ( & req -> version ) )
+ {
+ ext = false;
+ }
+
+ made |= VPathMakeOrNot ( & self -> http,
+ & src -> hUrl , & src -> ticket, src, ext, & rc, true );
+ made |= VPathMakeOrNot ( & self -> fasp,
+ & src -> fpUrl, & src -> ticket, src, ext, & rc, true );
+ made |= VPathMakeOrNot ( & self -> https,
+ & src -> hsUrl, & src -> ticket, src, ext, & rc, true );
+ made |= VPathMakeOrNot ( & self -> file,
+ & src -> flUrl, & src -> ticket, src, ext, & rc, true );
+ made |= VPathMakeOrNot ( & self -> s3,
+ & src -> s3Url, & src -> ticket, src, ext, & rc, true );
+ VPathMakeOrNot ( & self -> vcHttp,
+ & src -> hVdbcacheUrl, & src -> ticket, src, ext, & rc, false );
+ VPathMakeOrNot ( & self -> vcFasp,
+ & src -> fpVdbcacheUrl,& src -> ticket, src, ext, & rc, false );
+ VPathMakeOrNot ( & self -> vcHttps,
+ & src -> hsVdbcacheUrl,& src -> ticket, src, ext, & rc, false );
+ VPathMakeOrNot ( & self -> vcFile,
+ & src -> flVdbcacheUrl,& src -> ticket, src, ext, & rc, false );
+ VPathMakeOrNot ( & self -> vcS3,
+ & src -> s3VdbcacheUrl,& src -> ticket, src, ext, & rc, false );
+
+ if ( rc == 0 )
+ rc = EVPathInitMapping ( self, src, & req -> version );
+ return rc;
+ }
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcError, rcUnexpected );
+ break;
+
+ case 3:
+ /* redirection
+ currently this is being handled by our request object */
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcError, rcUnexpected );
+ break;
+ case 4:
+ /* client error */
+ lvl = klogErr;
+ switch ( src -> code )
+ {
+ case 400:
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcMessage, rcInvalid );
+ break;
+ case 401:
+ case 403:
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcQuery, rcUnauthorized );
+ break;
+ case 404: /* 404|no data :
+ If it is a real response then this assession is not found.
+ What if it is a DB failure? Will be retried if configured to do so? */
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcName, rcNotFound );
+ break;
+ case 410:
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcName, rcNotFound );
+ break;
+ default:
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcError, rcUnexpected );
+ }
+ break;
+ case 5:
+ /* server error */
+ lvl = klogSys;
+ switch ( src -> code )
+ {
+ case 503:
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcDatabase, rcNotAvailable );
+ break;
+ case 504:
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcTimeout, rcExhausted );
+ break;
+ default:
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcError, rcUnexpected );
+ }
+ break;
+ default:
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcError, rcUnexpected );
+ }
+
+ /* log message to user */
+ if ( req -> errorsToIgnore == 0 ) {
+ if ( src -> objectId . size > 0 )
+ PLOGERR ( lvl, ( lvl, rc,
+ "failed to resolve accession '$(acc)' - $(msg) ( $(code) )",
+ "acc=%S,msg=%S,code=%u",
+ & src -> objectId, & src -> message, src -> code ) );
+ else
+ PLOGERR ( lvl, ( lvl, rc,
+ "failed to resolve accession '$(acc)' - $(msg) ( $(code) )",
+ "acc=%s,msg=%S,code=%u", acc, & src -> message, src -> code ) );
+ }
+ else
+ -- ( ( SRequest * ) req ) -> errorsToIgnore;
+
+ return KSrvErrorMake ( & self -> error, src, rc );
+}
+
+
+static rc_t EVPathFini ( EVPath * self ) {
+ rc_t rc = 0;
+
+ assert ( self );
+
+ RELEASE ( VPath, self -> mapping );
+ RELEASE ( VPath, self -> http );
+ RELEASE ( VPath, self -> fasp );
+ RELEASE ( VPath, self -> https );
+ RELEASE ( VPath, self -> file );
+ RELEASE ( VPath, self -> s3 );
+ RELEASE ( VPath, self -> vcHttp );
+ RELEASE ( VPath, self -> vcFasp );
+ RELEASE ( VPath, self -> vcHttps );
+ RELEASE ( VPath, self -> vcFile );
+ RELEASE ( VPath, self -> vcS3 );
+
+ RELEASE ( KSrvError, self -> error );
+
+ return rc;
+}
+
+
+/* SRow ***********************************************************************/
+static rc_t SRowWhack ( void * p ) {
+ rc_t rc = 0;
+ rc_t r2 = 0;
+ if ( p != NULL ) {
+ SRow * self = ( SRow * ) p;
+ r2 = EVPathFini ( & self -> path );
+ if ( rc == 0) {
+ rc = r2;
+ }
+ r2 = SRawFini ( & self -> raw );
+ if ( rc == 0) {
+ rc = r2;
+ }
+ memset ( self, 0, sizeof * self );
+ free ( self );
+ }
+ return rc;
+}
+
+
+static rc_t SRowMake ( SRow ** self, const String * src, const SRequest * req,
+ const SConverters * f, const SVersion * version )
+{
+ rc_t rc = 0;
+ rc_t r2 = 0;
+
+ SRow * p = ( SRow * ) calloc ( 1, sizeof * p );
+ if ( p == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+
+ assert ( req && version && src );
+
+ rc = SRawAlloc ( & p -> raw, src -> addr, src -> size );
+ if ( rc == 0 ) {
+ assert ( f );
+ rc = SOrderedInit ( & p -> ordered, & p -> raw, f -> n );
+ }
+ if ( rc == 0 ) {
+ rc = STypedInit ( & p -> typed, & p -> ordered, f, version );
+ }
+ if ( rc == 0 &&
+ p -> typed . code == 200 && req -> request . objects == 1 )
+ {
+ String acc;
+ size_t l
+ = string_measure ( req -> request . object [ 0 ] . objectId, NULL );
+ StringInit ( & acc, req -> request . object [ 0 ] . objectId, l, l );
+ if ( ! StringEqual ( & p -> typed . accession, & acc ) &&
+ ! StringEqual ( & p -> typed . objectId , & acc ) )
+ {
+ return RC ( rcVFS, rcQuery, rcResolving, rcMessage, rcCorrupt );
+ }
+ }
+ if ( rc == 0 ) {
+ const char * acc = NULL;
+ if ( req -> request . objects == 1 ) {
+ acc = req -> request . object [ 0 ] . objectId;
+ }
+ rc = EVPathInit ( & p -> path, & p -> typed, req, & r2, acc );
+ if ( rc == 0 ) {
+ rc = r2;
+ }
+ }
+
+/* compare ticket
+ currently this makes sense with 1 request from a known workspace *
+ if ( download_ticket . size != 0 )
+ {
+ if ( ticket == NULL || ! StringEqual ( & download_ticket, ticket ) )
+ return RC ( rcVFS, rcQuery, rcResolving, rcMessage, rcCorrupt );
+ }
+*/
+
+ if ( rc == 0 )
+ rc = VPathSetMake ( & p -> set, & p -> path,
+ SVersionSingleUrl ( version ) );
+ if ( rc == 0 ) {
+ assert ( self );
+ * self = p;
+ }
+ else
+ SRowWhack ( p );
+
+ return rc;
+}
+
+
+static void whackSRow ( void * self, void * ignore ) {
+ SRowWhack ( self);
+}
+
+
+/* STimestamp ****************************************************************/
+static rc_t STimestampInit ( STimestamp * self, const String * src ) {
+ rc_t rc = 0;
+
+ assert ( self && src );
+
+ rc = SRawAlloc ( & self -> raw, src -> addr, src -> size );
+
+ if ( rc == 0 )
+ rc = KTimeInit ( & self -> time, src );
+
+ return rc;
+}
+
+
+static rc_t STimestampInitCurrent ( STimestamp * self ) {
+ assert ( self );
+
+ self -> time = KTimeStamp ();
+
+ if ( self -> time != 0 ) {
+ const size_t s = 32;
+ self -> raw . s = calloc ( 1, s );
+ if ( self -> raw . s == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ else {
+ size_t sz = KTimeIso8601 ( self -> time, self -> raw . s, s );
+ if ( sz == 0 )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory,
+ rcInsufficient );
+ else
+ return 0;
+ }
+ }
+ else
+ return RC ( rcVFS, rcQuery, rcExecuting, rcItem, rcIncorrect );
+}
+
+
+static rc_t STimestampFini ( STimestamp * self ) {
+ rc_t rc = 0;
+
+ assert ( self );
+
+ rc = SRawFini ( & self -> raw );
+
+ memset ( self, 0, sizeof * self );
+
+ return rc;
+}
+
+
+/* SServerTimestamp************************************************************/
+static
+rc_t SServerTimestampInit ( SServerTimestamp * self,
+ const String * src )
+{
+ rc_t rc = 0;
+ rc_t r2 = 0;
+
+ assert ( self );
+
+ rc = STimestampInit ( & self -> server, src );
+
+ r2 = STimestampInitCurrent ( & self -> local );
+ if ( rc == 0 )
+ rc = r2;
+
+ return rc;
+}
+
+
+static rc_t SServerTimestampFini ( SServerTimestamp * self ) {
+ rc_t rc = 0;
+ rc_t r2 = 0;
+
+ assert ( self );
+ rc = STimestampFini ( & self ->server );
+
+ r2 = STimestampFini ( & self ->local );
+ if ( rc == 0 )
+ rc = r2;
+
+ return rc;
+}
+
+
+/* SResponse ******************************************************************/
+static rc_t SResponseInit ( SResponse * self ) {
+ rc_t rc = 0;
+ assert ( self );
+ VectorInit ( & self -> rows, 0, 5 );
+ rc = KSrvResponseMake ( & self -> list );
+ return rc;
+}
+
+
+static void whackKartItem ( void * self, void * ignore ) {
+ KartItemRelease ( ( KartItem * ) self);
+}
+
+
+static rc_t SResponseFini ( SResponse * self ) {
+ rc_t rc = 0;
+ rc_t r2 = 0;
+
+ assert ( self );
+
+ {
+ void ( CC * whack ) ( void *item, void *data ) = NULL;
+ if ( self -> serviceType == eSTsearch )
+ whack = whackKartItem;
+ else
+ whack = whackSRow;
+ assert ( whack );
+ VectorWhack ( & self -> rows, whack, NULL );
+ }
+
+ rc = SHeaderFini ( & self -> header );
+
+ r2 = KSrvResponseRelease ( self -> list );
+ if ( rc == 0 )
+ rc = r2;
+
+ r2 = KartRelease ( self -> kart );
+ if ( rc == 0 )
+ rc = r2;
+
+ r2 = SServerTimestampFini ( & self -> timestamp );
+ if ( rc == 0 )
+ rc = r2;
+
+ memset ( self, 0, sizeof * self );
+
+ return rc;
+}
+
+
+static rc_t SResponseGetResponse
+ ( const SResponse * self, const KSrvResponse ** response )
+{
+ rc_t rc = 0;
+ assert ( self );
+ rc = KSrvResponseAddRef ( self -> list );
+ if ( rc == 0 ) {
+ * response = self -> list;
+ }
+ return rc;
+}
+
+
+/* SKV ************************************************************************/
+static
+rc_t SKVMake ( const SKV ** self, const char * k, const char * v )
+{
+ assert ( self );
+ * self = NULL;
+ if ( k == NULL || * k == '\0' ) {
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+ }
+ else {
+ rc_t rc = 0;
+ size_t num_writ = 0;
+ size_t sk = string_size ( k );
+ size_t sv = string_size ( v );
+ size_t s = sk + sv + 2;
+ char * p = ( char * ) malloc ( s );
+ if ( p == NULL ) {
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ }
+ rc = string_printf ( p, s, & num_writ, "%s=%s", k, v );
+ assert ( num_writ <= s );
+ if ( rc != 0 ) {
+ free ( p );
+ p = NULL;
+ }
+ else {
+ SKV * kv = ( SKV * ) malloc ( sizeof * kv );
+ assert ( sk );
+ if ( kv == NULL ) {
+ free ( p );
+ p = NULL;
+ rc = RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ }
+ else {
+ StringInit ( & kv -> k, p, sk, sk );
+ StringInit ( & kv -> v, p + sk + 1, sv, sv );
+ * self = kv;
+ }
+ }
+ return rc;
+ }
+}
+
+
+static
+rc_t SKVMakeObj ( const SKV ** self, const SObject * obj,
+ const SVersion * version )
+{
+ rc_t rc = 0;
+ size_t sk = 0;
+ size_t num_writ = 0;
+ char tmp [] = "";
+ bool old = SVersionAccInRequest ( version );
+ char * p = NULL;
+ const char * k = "object";
+ if ( old )
+ k = "acc";
+
+ sk = string_size ( k );
+
+ assert ( self && obj );
+ * self = NULL;
+
+ if ( old )
+ rc = string_printf ( tmp, 1, & num_writ, "%s=%s", k,
+ obj -> objectId );
+ else
+ string_printf ( tmp, 1, & num_writ, "%s=%d|%s|%s", k, obj -> ordId,
+ ObjectTypeToString ( obj -> objectType ), obj -> objectId );
+
+ ++ num_writ;
+ p = ( char * ) malloc ( num_writ );
+ if ( p == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+
+ if ( old )
+ rc = string_printf ( p, num_writ, & num_writ, "%s=%s", k,
+ obj -> objectId );
+ else
+ rc = string_printf ( p, num_writ, & num_writ, "%s=%d|%s|%s", k,
+ obj -> ordId, ObjectTypeToString ( obj -> objectType ),
+ obj -> objectId );
+
+ if ( rc != 0 ) {
+ free ( p );
+ p = NULL;
+ }
+ else {
+ SKV * kv = ( SKV * ) malloc ( sizeof * kv );
+ assert ( sk );
+ if ( kv == NULL ) {
+ free ( p );
+ p = NULL;
+ rc = RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ }
+ else {
+ -- num_writ;
+ StringInit ( & kv -> k, p, sk, sk );
+ StringInit ( & kv -> v, p + sk + 1, num_writ, num_writ );
+ * self = kv;
+ }
+ }
+
+ return rc;
+}
+
+
+/* SHttpRequestHelper ********************************************************/
+static rc_t SHttpRequestHelperInit ( SHttpRequestHelper * self,
+ const KNSManager * kMgr, const char * cgi )
+{
+ rc_t rc = 0;
+
+ assert ( self );
+
+ memset ( self, 0, sizeof * self );
+
+ rc = KNSManagerMakeReliableClientRequest ( kMgr, & self -> httpReq,
+ 0x01010000, NULL, cgi );
+
+ return rc;
+}
+
+
+static rc_t SHttpRequestHelperFini ( SHttpRequestHelper * self ) {
+ rc_t rc = 0;
+
+ assert ( self );
+
+ RELEASE ( KHttpRequest, self -> httpReq );
+
+ return rc;
+}
+
+
+static
+void SHttpRequestHelperAddPostParam ( void * item, void * data )
+{
+ const SKV * kv = ( SKV * ) item;
+ SHttpRequestHelper * p = ( SHttpRequestHelper * ) data;
+
+ rc_t rc = 0;
+
+ assert ( kv && p );
+
+ rc = KHttpRequestAddPostParam ( p -> httpReq, kv -> k . addr );
+ if ( p -> rc == 0 )
+ p -> rc = rc;
+}
+
+
+/* SCgiRequest ****************************************************************/
+static
+rc_t SCgiRequestInitCgi ( SCgiRequest * self, const char * cgi )
+{
+ assert ( self && ! self -> cgi );
+
+ self -> cgi = string_dup_measure ( cgi, NULL );
+ if ( self -> cgi == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ return 0;
+}
+
+
+static void whackSKV ( void * p, void * ignore ) {
+ SKV * self = ( SKV * ) p;
+ assert ( self );
+ free ( ( void * ) self -> k . addr );
+ memset ( self, 0, sizeof * self );
+ free ( self );
+}
+
+
+static void SCgiRequestFini ( SCgiRequest * self ) {
+ assert ( self );
+ free ( self -> cgi );
+ VectorWhack ( & self -> params, whackSKV, NULL );
+ memset ( self, 0, sizeof * self );
+}
+
+
+static rc_t SCgiRequestPerform ( const SCgiRequest * self,
+ const SHelper * helper, KStream ** response,
+ const char * expected )
+{
+ rc_t rc = 0;
+
+ assert ( self && helper && response );
+
+ if ( rc == 0 ) {
+ SHttpRequestHelper h;
+ rc = SHttpRequestHelperInit ( & h, helper -> kMgr, self-> cgi );
+ if ( rc == 0 ) {
+ VectorForEach (
+ & self -> params, false, SHttpRequestHelperAddPostParam, & h );
+ rc = h . rc;
+ }
+
+ if ( rc == 0 ) {
+ if ( expected == NULL ) {
+ KHttpResult * rslt = NULL;
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ), (
+ ">>>>>>>>>>>>>>>> sending HTTP POST request >>>>>>>>>>>>>>>>\n" ) );
+ rc = KHttpRequestPOST ( h . httpReq, & rslt );
+ if ( rc == 0 ) {
+ uint32_t code = 0;
+ rc = KHttpResultStatus ( rslt, & code, NULL, 0, NULL );
+ if ( rc == 0 ) {
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ), (
+ " code=%d\n", code ) );
+ switch ( code ) {
+ case 200:
+ rc = KHttpResultGetInputStream ( rslt, response );
+ break;
+ case 403:
+ /* HTTP/1.1 403 Forbidden
+ - resolver CGI was called over http insted of https */
+ rc = RC ( rcVFS, rcQuery, rcExecuting, rcConnection,
+ rcUnauthorized );
+ break;
+ case 404:
+ /* HTTP/1.1 404 Bad Request - resolver CGI was not found */
+ rc = RC ( rcVFS, rcQuery, rcExecuting, rcConnection,
+ rcNotFound );
+ break;
+ default: /* Something completely unexpected */
+ rc = RC ( rcVFS, rcQuery, rcExecuting, rcConnection,
+ rcUnexpected );
+ break;
+ }
+ }
+ }
+ RELEASE ( KHttpResult, rslt );
+ }
+ else {
+ KStream * stream = NULL;
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ), (
+ "XXXXXXXXXXXX NOT sending HTTP POST request XXXXXXXXXXXXXXXX\n" ) );
+ rc = KStreamMakeFromBuffer ( & stream, expected,
+ string_size ( expected ) );
+ if ( rc == 0 )
+ * response = stream;
+ }
+ }
+
+ {
+ rc_t r2 = SHttpRequestHelperFini ( & h );
+ if ( rc == 0 )
+ rc = r2;
+ }
+ }
+
+ return rc;
+}
+
+
+/* SObject ********************************************************************/
+static rc_t SObjectInit ( SObject * self,
+ const char * objectId, size_t objSz, EObjectType objectType )
+{
+ assert ( self );
+ self -> objectType = objectType;
+ if ( objectId != NULL && objSz != 0 ) {
+ self -> objectId = string_dup ( objectId, objSz );
+ if ( self -> objectId == NULL ) {
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ }
+ }
+ return 0;
+}
+
+
+static void SObjectFini ( SObject * self ) {
+ assert ( self );
+ free ( self -> objectId );
+ memset ( self, 0, sizeof * self );
+}
+
+
+/* SRequestData ***************************************************************/
+/*static
+rc_t SRequestDataInit ( SRequestData * self, const char * acc, size_t acc_sz,
+ EObjectType objectType )
+{
+ rc_t rc = 0;
+ assert ( self );
+ memset ( self, 0, sizeof * self );
+ if ( acc != NULL && acc_sz != 0 ) {
+ self -> objects = 1;
+ rc = SObjectInit ( & self -> object [ 0 ], acc, acc_sz, objectType );
+ }
+ return rc;
+}*/
+
+
+static void SRequestDataFini ( SRequestData * self ) {
+ uint32_t i = 0;
+ assert ( self );
+ for ( i = 0; i < self -> objects; ++i ) {
+ SObjectFini ( & self -> object [ i ] );
+ }
+ memset ( self, 0, sizeof * self );
+}
+
+
+static
+rc_t SRequestDataAppendObject ( SRequestData * self, const char * id,
+ size_t id_sz, EObjectType objectType )
+{
+ rc_t rc = 0;
+ assert ( self );
+ if ( self -> objects > sizeof self -> object / sizeof self -> object [ 0 ] )
+ {
+ return RC ( rcVFS, rcQuery, rcExecuting, rcItem, rcExcessive );
+ }
+ if ( id_sz == 0 )
+ id_sz = string_measure ( id, NULL );
+ rc = SObjectInit ( & self -> object [ self -> objects ], id, id_sz,
+ objectType );
+ if ( rc == 0 ) {
+ ++ self -> objects;
+ }
+ return rc;
+}
+
+
+/* BSTItem *******************************************************************/
+static int64_t CC BSTItemCmp ( const void * item, const BSTNode * n ) {
+ const String * s = item;
+ const BSTItem * i = ( BSTItem * ) n;
+
+ assert ( s && i );
+
+ return string_cmp ( s -> addr, s -> size,
+ i -> ticket, string_measure ( i -> ticket, NULL ), s -> size );
+}
+
+static int64_t CC BSTreeSort ( const BSTNode * item, const BSTNode * n ) {
+ const BSTItem * i = ( BSTItem * ) item;
+ String str;
+ size_t size = 0;
+ uint32_t len = string_measure ( i -> ticket, & size );
+ StringInit ( & str, i -> ticket, size, len );
+ return BSTItemCmp ( & str, n );
+}
+
+static void BSTItemWhack ( BSTNode * n, void * ignore ) {
+ BSTItem * i = ( BSTItem * ) n;
+ assert ( i );
+ free ( i -> ticket );
+ memset ( i, 0, sizeof * i );
+ free ( i );
+}
+
+/* STickets *******************************************************************/
+const uint64_t BICKETS = 1024;
+static rc_t STicketsAppend ( STickets * self, uint32_t project,
+ const char * ticket )
+{
+ rc_t rc = 0;
+
+ const char * comma = "";
+
+ assert ( self );
+
+ if ( ticket == NULL )
+ return 0;
+
+ if ( rc == 0 && project > 0 && ticket [ 0 ] ) {
+ BSTItem * i = NULL;
+
+ String str;
+ size_t size = 0;
+ uint32_t len = string_measure ( ticket, & size );
+ StringInit ( & str, ticket, size, len );
+
+ i = ( BSTItem * ) BSTreeFind
+ ( & self -> ticketsToProjects, & str, BSTItemCmp );
+ if ( i != NULL )
+ return 0;
+
+ i = calloc ( 1, sizeof * i );
+ if ( i != NULL )
+ i -> ticket = string_dup_measure ( ticket, NULL );
+ if ( i == NULL || i -> ticket == NULL ) {
+ free ( i );
+ return RC ( rcVFS, rcStorage, rcAllocating, rcMemory, rcExhausted );
+ }
+
+ i -> project = project;
+
+ rc = BSTreeInsert ( & self -> ticketsToProjects, ( BSTNode * ) i,
+ BSTreeSort );
+ }
+
+ if ( self -> size > 0 )
+ comma = ",";
+
+ do {
+ size_t num_writ = 0;
+ char * p = ( char * ) self -> str . base;
+ assert ( comma );
+ rc = string_printf ( p + self -> size,
+ self -> str . elem_count - self -> size, & num_writ,
+ "%s%s", comma, ticket );
+ if ( rc == 0 ) {
+ rc_t r2 = 0;
+ String * s = ( String * ) malloc ( sizeof * s );
+ if ( s == NULL )
+ rc = RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ else {
+ const char * addr = p + self -> size;
+ uint32_t len = num_writ;
+ if ( comma [ 0 ] != '\0' ) {
+ ++ addr;
+ -- len;
+ }
+ StringInit ( s, addr, len, len );
+ r2 = VectorAppend ( & self -> tickets, NULL, s );
+ if ( r2 != 0 ) {
+ rc = r2;
+ free ( s );
+ }
+ self -> size += num_writ;
+ break;
+ }
+ }
+ else if ( GetRCState ( rc ) == rcInsufficient
+ && GetRCObject ( rc ) == ( enum RCObject ) rcBuffer )
+ {
+ size_t needed = BICKETS;
+ if ( self -> str . elem_count - self -> size + needed < num_writ )
+ needed = num_writ;
+ rc = KDataBufferResize
+ ( & self -> str, self -> str . elem_count + needed );
+ }
+ else
+ break;
+ } while ( rc == 0 );
+
+ return rc;
+}
+
+
+/*static void STicketsAppendFromVector ( void * item, void * data ) {
+ STickets * self = ( STickets * ) data;
+ rc_t rc = STicketsAppend ( self, ( char * ) item );
+ if ( rc != 0 ) {
+ self -> rc = rc;
+ }
+}*/
+
+
+static rc_t STicketsInit ( STickets * self ) {
+ assert ( self );
+ memset ( self, 0, sizeof * self );
+ BSTreeInit ( & self -> ticketsToProjects );
+ return KDataBufferMakeBytes ( & self -> str, BICKETS );
+}
+
+
+static void whack_free ( void * self, void * ignore ) {
+ if ( self != NULL ) {
+ memset ( self, 0, sizeof ( * ( char * ) self ) );
+ free ( self );
+ }
+}
+
+static rc_t STicketsFini ( STickets * self ) {
+ rc_t rc = 0;
+
+ assert ( self );
+
+ rc = KDataBufferWhack ( & self -> str );
+ VectorWhack ( & self -> tickets, whack_free, NULL );
+ BSTreeWhack ( & self -> ticketsToProjects, BSTItemWhack, NULL );
+
+ memset ( self, 0 , sizeof * self );
+
+ return rc;
+}
+
+
+/* Tickets ********************************************************************/
+typedef struct {
+ Vector * v;
+ rc_t rc;
+} Tickets;
+
+
+static void TicketsAppendTicket ( void * item, void * data ) {
+ const String * ticket = ( String * ) item;
+ Tickets * v = ( Tickets * ) data;
+ const SKV * kv = NULL;
+ const char k [] = "tic";
+ char * c = string_dup ( ticket -> addr, ticket -> size );
+ if ( c == NULL ) {
+ v -> rc = RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ return;
+ }
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ), ( " %s=%s\n",
+ k, c ) );
+ {
+ rc_t rc = SKVMake ( & kv, k, c );
+ free ( c );
+ if ( rc == 0 ) {
+ rc = VectorAppend ( v -> v, NULL, kv );
+ if ( rc != 0 && v -> rc == 0)
+ v -> rc = rc;
+ }
+ }
+}
+
+
+/* SRequest *******************************************************************/
+static rc_t SRequestInit ( SRequest * self ) {
+ assert ( self );
+
+ memset ( self, 0, sizeof * self );
+
+ return STicketsInit ( & self -> tickets );
+}
+
+
+static rc_t SRequestReset ( SRequest * self ) {
+ rc_t rc = 0;
+ rc_t r2 = 0;
+
+ assert ( self );
+
+ r2 = SVersionFini ( & self -> version );
+ if ( rc == 0 )
+ rc = r2;
+
+ SRequestDataFini ( & self -> request );
+ SCgiRequestFini ( & self -> cgiReq );
+
+ return rc;
+}
+
+
+static rc_t SRequestFini ( SRequest * self ) {
+ rc_t r2 = 0;
+ rc_t rc = SRequestReset ( self );
+
+ assert ( self );
+
+ r2 = STicketsFini ( & self -> tickets );
+ if ( rc == 0 )
+ rc = r2;
+
+ memset ( self, 0, sizeof * self );
+
+ return rc;
+}
+
+
+static rc_t SRequestAddTicket ( SRequest * self, uint32_t project,
+ const char * ticket )
+{
+ assert ( self );
+ return STicketsAppend ( & self -> tickets, project, ticket );
+}
+
+
+static
+rc_t SRequestInitNamesSCgiRequest ( SRequest * request, SHelper * helper,
+ VRemoteProtocols protocols, const char * cgi,
+ const char * version, bool aProtected )
+{
+ SCgiRequest * self = NULL;
+ rc_t rc = 0;
+ const SKV * kv = NULL;
+
+ assert ( request );
+
+ rc = SVersionFini ( & request -> version );
+ if ( rc != 0 )
+ return rc;
+
+ rc = SVersionInit ( & request -> version, version, eSTnames );
+ if ( rc != 0 )
+ return rc;
+
+ if ( protocols == eProtocolDefault )
+ protocols = SHelperDefaultProtocols ( helper );
+ request -> protocols = protocols;
+
+ self = & request -> cgiReq;
+
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ), (
+ "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" ) );
+
+ if ( self -> cgi == NULL ) {
+ char buffer [ 1024 ] = "";
+ if ( cgi == NULL ) {
+ rc = SHelperResolverCgi ( helper, aProtected,
+ buffer, sizeof buffer );
+ cgi = buffer;
+ }
+ rc = SCgiRequestInitCgi ( self, cgi );
+ }
+
+ VectorWhack ( & self -> params, whackSKV, NULL );
+
+ VectorInit ( & self -> params, 0, 5 );
+
+ request -> serviceType = eSTnames;
+ DBGMSG ( DBG_VFS,
+ DBG_FLAG ( DBG_VFS_SERVICE ), ( "CGI = '%s'\n", self -> cgi ) );
+ if ( rc == 0 ) {
+ const char name [] = "version";
+ char * version = NULL;
+ rc = SVersionToString ( & request -> version, & version );
+ if ( rc != 0 ) {
+ return rc;
+ }
+ rc = SKVMake ( & kv, name, version );
+ if ( rc == 0 ) {
+ rc = VectorAppend ( & self -> params, NULL, kv );
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ),
+ ( " %s=%s\n", name, version ) );
+ }
+ free ( version );
+ if ( rc != 0 )
+ return rc;
+ }
+ if ( ! SVersionHasMultpileObjects ( & request -> version ) ) {
+ if ( request -> request . object [ 0 ] . objectId == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+ else {
+ const char name [] = "acc";
+ rc = SKVMake
+ ( & kv, name, request -> request . object [ 0 ] . objectId );
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ), ( " %s=%s\n",
+ name, request -> request . object [ 0 ] . objectId ) );
+ if ( rc == 0 )
+ rc = VectorAppend ( & self -> params, NULL, kv );
+ }
+ if ( rc != 0 )
+ return rc;
+ }
+ else {
+ uint32_t i = 0;
+ for ( i = 0; i < request -> request . objects; ++i ) {
+ request -> request . object [ i ] . ordId = i;
+ rc = SKVMakeObj ( & kv, & request -> request . object [ i ],
+ & request -> version );
+ if ( rc == 0 ) {
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ),
+ ( " %.*s=%.*s\n", kv -> k . len, kv -> k . addr,
+ kv -> v . len, kv -> v . addr ) );
+ rc = VectorAppend ( & self -> params, NULL, kv );
+ }
+ }
+ if ( rc != 0 )
+ return rc;
+ }
+ if ( request -> tickets . size != 0 ) { /* optional */
+ Tickets t = { & self -> params, 0 };
+ VectorForEach ( & request -> tickets .tickets , false,
+ TicketsAppendTicket, & t );
+ rc = t . rc;
+ if ( rc != 0 )
+ return rc;
+ }
+ {
+ uint32_t i = 0;
+ const char * prefs [ eProtocolMaxPref ];
+ const char * seps [ eProtocolMaxPref ];
+ VRemoteProtocols protos = protocols;
+
+ prefs [ 0 ] = seps [ 0 ] = NULL;
+ prefs [ 1 ] = seps [ 1 ] = prefs [ 2 ] = seps [ 2 ]
+ = prefs [ 3 ] = seps [ 3 ] = "";
+
+ for ( i = 0; protos != 0 && i < sizeof prefs / sizeof prefs [ 0 ];
+ protos >>= 3 )
+ {
+ /* 1.1 protocols */
+ switch ( protos & eProtocolMask )
+ {
+ case eProtocolHttp:
+ prefs [ i ] = "http";
+ seps [ i ++ ] = ",";
+ break;
+ case eProtocolFasp:
+ prefs [ i ] = "fasp";
+ seps [ i ++ ] = ",";
+ break;
+ /* 1.2 protocols */
+ case eProtocolHttps:
+ prefs [ i ] = "https";
+ seps [ i ++ ] = ",";
+ break;
+ /* 3.0 protocols */
+ case eProtocolFile:
+ prefs [ i ] = "file";
+ seps [ i ++ ] = ",";
+ break;
+ default:
+ assert ( 0 );
+ break;
+ }
+ }
+ if ( prefs [ 0 ] == NULL )
+ rc = RC ( rcVFS, rcQuery, rcResolving, rcParam, rcInvalid );
+ else
+ {
+ const char name [] = "accept-proto";
+ size_t num_writ = 0;
+ char p [ 512 ] = "";
+ rc = string_printf ( p, sizeof p, & num_writ, "%s%s%s%s%s%s",
+ prefs [ 0 ], seps [ 1 ], prefs [ 1 ], seps [ 2 ], prefs [ 2 ],
+ seps [ 3 ], prefs [ 3 ] );
+ rc = SKVMake ( & kv, name, p );
+ if ( rc == 0 ) {
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ),
+ ( " %s=%s\n", name, p ) );
+ rc = VectorAppend ( & self -> params, NULL, kv );
+ }
+ }
+ if ( rc != 0 ) {
+ return rc;
+ }
+ }
+ if ( SVersionHasRefseqCtx ( & request -> version ) &&
+ request -> request . refseq_ctx )
+ {
+ const char name [] = "ctx";
+ rc = SKVMake ( & kv, name, "refseq" );
+ if ( rc == 0 ) {
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ),
+ ( " %s=refseq\n", name ) );
+ rc = VectorAppend ( & self -> params, NULL, kv );
+ }
+ if ( rc != 0 ) {
+ return rc;
+ }
+ }
+ if ( SVersionTypInRequest ( & request -> version ) ) {
+ if ( request -> request . object [ 0 ] . objectType !=
+ eOT_undefined )
+ {
+ const char name [] = "typ";
+ const char * v = ObjectTypeToString
+ ( request -> request . object [ 0 ] . objectType );
+ rc = SKVMake ( & kv, name, v );
+ if ( rc == 0 ) {
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ),
+ ( " %s=%s\n", name, v ) );
+ rc = VectorAppend ( & self -> params, NULL, kv );
+ }
+ }
+ if ( rc != 0 ) {
+ return rc;
+ }
+ }
+ return rc;
+}
+
+
+static
+rc_t SRequestInitSearchSCgiRequest ( SRequest * request, const char * cgi,
+ const char * version )
+{
+ SCgiRequest * self = NULL;
+ rc_t rc = 0;
+ const SKV * kv = NULL;
+ assert ( request );
+ rc = SVersionInit ( & request -> version, version, eSTnames );
+ if ( rc != 0 )
+ return rc;
+ self = & request -> cgiReq;
+ if ( self -> cgi == NULL ) {
+ if ( cgi == NULL ) {
+/* try to get cgi from kfg, otherwise use hardcoded below */
+ cgi = "http://www.ncbi.nlm.nih.gov/Traces/names/search.cgi";
+ cgi = "https://www.ncbi.nlm.nih.gov/Traces/names/search.cgi";
+ }
+ rc = SCgiRequestInitCgi ( self, cgi );
+ }
+ request -> serviceType = eSTsearch;
+ VectorInit ( & self -> params, 0, 5 );
+ DBGMSG ( DBG_VFS,
+ DBG_FLAG ( DBG_VFS_SERVICE ), ( "CGI = '%s'\n", self -> cgi ) );
+ if ( rc == 0 ) {
+ const char name [] = "version";
+ char * version = NULL;
+ rc = SVersionToString ( & request -> version, & version );
+ if ( rc != 0 ) {
+ return rc;
+ }
+ rc = SKVMake ( & kv, name, version );
+ if ( rc == 0 ) {
+ rc = VectorAppend ( & self -> params, NULL, kv );
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ),
+ ( " %s=%s\n", name, version ) );
+ }
+ free ( version );
+ if ( rc != 0 ) {
+ return rc;
+ }
+ }
+ {
+ const char name [] = "term";
+ char * b = NULL;
+ uint32_t i = 0;
+ size_t l = 0;
+ size_t o = 0;
+ for ( i = 0; i < request -> request . objects; ++i ) {
+ l += string_measure
+ ( request -> request . object [ i ] . objectId, NULL ) + 1;
+ }
+ if ( l > 0 ) {
+ b = ( char * ) malloc ( l );
+ if ( b == NULL ) {
+ return
+ RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+ }
+ for ( i = 0; rc == 0 && i < request -> request . objects;
+ ++i )
+ {
+ size_t num_writ = 0;
+ rc = string_printf ( b + o, l - o, & num_writ,
+ "%s", request -> request . object [ i ] );
+ o += num_writ;
+ if ( i + 1 == request -> request . objects ) {
+ b [ o ++ ] = '\0';
+ }
+ else {
+ b [ o ++ ] = ',';
+ }
+ }
+ assert ( o <= l );
+ rc = SKVMake ( & kv, name, b );
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ),
+ ( " %s=%s\n", name, b ) );
+ if ( rc == 0 ) {
+ rc = VectorAppend ( & self -> params, NULL, kv );
+ }
+ free ( b );
+ if ( rc != 0 ) {
+ return rc;
+ }
+ }
+ }
+ return rc;
+}
+
+
+/* KService *******************************************************************/
+static void KServiceExpectErrors ( KService * self, int n ) {
+ assert ( self );
+
+ self -> req . errorsToIgnore = n;
+}
+
+
+static rc_t KServiceAddObject ( KService * self,
+ const char * id, size_t id_sz, EObjectType objectType )
+{
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ return SRequestDataAppendObject
+ ( & self -> req . request, id, id_sz, objectType );
+}
+
+
+/* Add an Id ( Accession or Object-Id ) to service request */
+rc_t KServiceAddId ( KService * self, const char * id ) {
+ return KServiceAddObject ( self, id, 0, eOT_undefined );
+}
+
+
+static rc_t KServiceAddTicket ( KService * self, const char * ticket ) {
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ if ( ticket == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ return SRequestAddTicket ( & self -> req, 0, ticket );
+}
+
+
+/* Add a dbGaP Project to service request */
+rc_t KServiceAddProject ( KService * self, uint32_t project ) {
+ rc_t rc = 0;
+
+ char buffer [ 256 ] = "";
+ size_t ticket_size = ~0;
+
+ if ( project == 0 )
+ return 0;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ rc = SHelperProjectToTicket ( & self -> helper, project,
+ buffer, sizeof buffer, & ticket_size );
+ if ( rc != 0 )
+ return rc;
+
+ assert ( ticket_size <= sizeof buffer );
+
+ return SRequestAddTicket ( & self -> req, project, buffer );
+}
+
+
+static
+rc_t KServiceInitNamesRequestWithVersion ( KService * self,
+ VRemoteProtocols protocols,
+ const char * cgi, const char * version, bool aProtected )
+{
+ assert ( self );
+
+ return SRequestInitNamesSCgiRequest ( & self -> req, & self -> helper,
+ protocols, cgi, version, aProtected );
+}
+
+
+static
+rc_t KServiceInitNamesRequest ( KService * self, VRemoteProtocols protocols,
+ const char * cgi )
+{
+ return KServiceInitNamesRequestWithVersion ( self, protocols, cgi, "#3.0",
+ false );
+}
+
+
+static
+rc_t KServiceInitSearchRequestWithVersion ( KService * self, const char * cgi,
+ const char * version )
+{
+ assert ( self );
+
+ return SRequestInitSearchSCgiRequest ( & self -> req, cgi, version );
+}
+
+
+static rc_t KServiceInit ( KService * self, const KNSManager * mgr ) {
+ rc_t rc = 0;
+
+ assert ( self );
+ memset ( self, 0, sizeof * self );
+
+ if ( rc == 0 )
+ rc = SHelperInit ( & self -> helper, mgr );
+
+ if ( rc == 0 )
+ rc = SResponseInit ( & self -> resp );
+
+ if ( rc == 0 )
+ rc = SRequestInit ( & self -> req );
+
+ return rc;
+}
+
+
+/* Initialize KService with a single "acc"/"objectType" and optional "ticket"
+ in Request */
+static rc_t KServiceInitNames1 ( KService * self, const KNSManager * mgr,
+ const char * cgi, const char * version, const char * acc,
+ size_t acc_sz, const char * ticket, VRemoteProtocols protocols,
+ EObjectType objectType, bool refseq_ctx, bool aProtected )
+{
+ rc_t rc = 0;
+
+ if ( rc == 0 )
+ rc = KServiceInit ( self, mgr );
+
+ if ( rc == 0 )
+ rc = KServiceAddObject ( self, acc, acc_sz, objectType );
+ if ( rc == 0 )
+ rc = SRequestAddTicket ( & self -> req, 0, ticket );
+ if ( rc == 0 )
+ self -> req . request . refseq_ctx = refseq_ctx;
+
+ if ( rc == 0 )
+ rc = KServiceInitNamesRequestWithVersion
+ ( self, protocols, cgi, version, aProtected );
+
+ return rc;
+}
+
+
+static
+rc_t KServiceMakeWithMgr ( KService ** self, const KNSManager * mgr )
+{
+ rc_t rc = 0;
+
+ KService * p = NULL;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ p = ( KService * ) calloc ( 1, sizeof * p );
+ if ( p == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcMemory, rcExhausted );
+
+ rc = KServiceInit ( p, mgr );
+ if ( rc == 0)
+ * self = p;
+ else
+ free ( p );
+
+ return rc;
+}
+
+
+/* Make KService object */
+rc_t KServiceMake ( KService ** self) {
+ return KServiceMakeWithMgr ( self, NULL );
+}
+
+
+static rc_t KServiceFini ( KService * self ) {
+ rc_t rc = 0;
+ rc_t r2 = 0;
+
+ assert ( self );
+
+ r2 = SResponseFini ( & self -> resp );
+ if ( rc == 0 )
+ rc = r2;
+
+ r2 = SRequestFini ( & self -> req );
+ if ( rc == 0 )
+ rc = r2;
+
+ r2 = SHelperFini ( & self -> helper );
+ if ( rc == 0 )
+ rc = r2;
+
+ return rc;
+}
+
+
+/* Release KService object */
+rc_t KServiceRelease ( KService * self ) {
+ rc_t rc = 0;
+
+ if ( self != NULL ) {
+ rc = KServiceFini ( self );
+ free ( self );
+ }
+
+ return rc;
+}
+
+
+static
+rc_t KServiceProcessLine ( KService * self,
+ const String * line, bool * end )
+{
+ rc_t rc = 0;
+ assert ( self && line && end );
+ if ( line -> addr [ 0 ] == '$' ) {
+ * end = true;
+ if ( SVersionResponseHasTimestamp
+ ( & self -> resp . header . version )
+ && line -> size > 2 && line -> len > 2 )
+ {
+ String timestamp;
+ StringInit ( & timestamp, line -> addr + 2, line -> size - 2,
+ line -> len - 2 );
+ rc = SServerTimestampInit ( & self -> resp . timestamp,
+ & timestamp );
+ }
+ }
+ else if ( self -> req . serviceType == eSTsearch ) {
+ const char str [] = "$end";
+ size_t sz = sizeof str - 1;
+ if ( string_cmp ( line -> addr, line -> size, str, sz, ( uint32_t ) sz )
+ == 0)
+ {
+ * end = true;
+ }
+ else
+ rc = KartAddRow ( self -> resp . kart, line -> addr, line -> size );
+ }
+ else {
+ const SConverters * f = NULL;
+ rc = SConvertersMake ( & f, & self -> resp . header );
+ if ( rc == 0 ) {
+ SRow * row = NULL;
+ rc_t r2 = SRowMake ( & row, line, & self -> req, f,
+ & self -> resp . header . version );
+ uint32_t l = VectorLength ( & self -> resp . rows );
+ if ( r2 == 0 ) {
+ if ( SVersionHasMultpileObjects
+ ( & self -> resp . header. version )
+ || l == 0 )
+ {
+ r2 = VectorAppend ( & self -> resp . rows, NULL, row );
+ }
+ else {
+ /* ACC.vdbcashe : TODO : search for vdb.cache extension */
+ if ( row -> typed . objectId . len == 18 ||
+ row -> typed . objectId . len == 19 )
+ {
+ r2 = SRowWhack ( row );
+ row = NULL;
+ }
+ }
+ }
+ if ( r2 == 0 ) {
+ if ( SVersionHasMultpileObjects
+ ( & self -> resp . header . version)
+ || KSrvResponseLength ( self -> resp . list ) == 0 )
+ {
+ r2 = KSrvResponseAppend ( self -> resp . list, row -> set );
+ }
+ else
+ assert ( ! row );
+ }
+ if ( r2 != 0 && rc == 0 && l != 1 )
+ rc = r2;
+ }
+ }
+ return rc;
+}
+
+
+static
+rc_t KServiceProcessStream ( KService * self, KStream * stream )
+{
+ bool start = true;
+ char buffer [ 4096 ] = "";
+ size_t num_read = 0;
+ timeout_t tm;
+ rc_t rc = 0;
+ size_t sizeW = sizeof buffer;
+ size_t sizeR = 0;
+ size_t offR = 0;
+ size_t offW = 0;
+ char * newline = NULL;
+ assert ( self );
+ self -> resp . serviceType = self -> req . serviceType;
+ rc = TimeoutInit ( & tm, self -> helper . timeoutMs );
+
+ if ( rc == 0 )
+ rc = SResponseFini ( & self -> resp );
+ if ( rc == 0 )
+ rc = SResponseInit ( & self -> resp );
+
+ if ( rc == 0 && self -> req . serviceType == eSTsearch )
+ rc = KartMake2 ( & self -> resp . kart );
+
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ), (
+ "-----------------------------------------------------------\n" ) );
+
+ while ( rc == 0 ) {
+ if ( sizeR == 0 ) {
+ rc = KStreamTimedRead
+ ( stream, buffer + offW, sizeW, & num_read, & tm );
+ if ( rc != 0 || num_read == 0 )
+ break;
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ),
+ ( "%.*s\n", ( int ) num_read - 1, buffer + offW ) );
+ sizeR += num_read;
+ offW += num_read;
+ if (sizeW >= num_read )
+ sizeW -= num_read;
+ else
+ sizeW = 0;
+ }
+ newline = string_chr ( buffer + offR, sizeR, '\n' );
+/* TODO different conditions: move buffer content; partial read; line > buf size
+ */
+ if ( newline == NULL ) {
+ if ( sizeW == 0 && offR == 0 ) {
+ rc = RC
+ ( rcVFS, rcQuery, rcExecuting, rcString, rcInsufficient );
+ break;
+ }
+ rc = KStreamTimedRead
+ ( stream, buffer + offW, sizeW, & num_read, & tm );
+ if ( rc != 0 ) {
+ /* TBD - look more closely at rc */
+ if ( num_read > 0 )
+ rc = 0;
+ else
+ break;
+ }
+ else if ( num_read == 0 ) {
+ rc = RC /* stream does not end by '\n' */
+ ( rcVFS, rcQuery, rcExecuting, rcString, rcInsufficient );
+ break;
+ }
+ sizeR += num_read;
+ offW += num_read;
+ if (sizeW >= num_read )
+ sizeW -= num_read;
+ else
+ sizeW = 0;
+ }
+ else {
+ size_t size = newline - ( buffer + offR );
+ String s;
+ s . addr = buffer + offR;
+ s . len = s . size = size;
+ if ( start ) {
+ rc = SHeaderMake
+ ( & self -> resp . header, & s, self -> req . serviceType );
+ if ( rc != 0 )
+ break;
+ start = false;
+ }
+ else {
+ bool end = false;
+ rc = KServiceProcessLine ( self, & s, & end );
+ if ( end || rc != 0 )
+ break;
+ }
+ ++ size;
+ offR += size;
+ if ( sizeR >= size )
+ sizeR -= size;
+ else
+ sizeR = 0;
+ if ( sizeR == 0 && offR == offW ) {
+ offR = offW = 0;
+ sizeW = sizeof buffer;
+ }
+ }
+ }
+ if ( start )
+ rc = RC ( rcVFS, rcQuery, rcExecuting, rcString, rcInsufficient );
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ),
+ ( "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ "
+ "rc = %R\n\n", rc ) );
+ return rc;
+}
+
+
+static
+rc_t KServiceGetResponse
+ ( const KService * self, const KSrvResponse ** response )
+{
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ if ( response == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+ else
+ return SResponseGetResponse ( & self -> resp, response );
+}
+
+
+rc_t KServiceGetConfig ( struct KService * self, const KConfig ** kfg) {
+ rc_t rc = 0;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+ if ( kfg == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ rc = SHelperInitKfg ( & self -> helper );
+ if ( rc == 0 )
+ rc = KConfigAddRef ( self -> helper . kfg );
+
+ if ( rc == 0 )
+ * kfg = self -> helper . kfg;
+
+ return rc;
+}
+
+rc_t KServiceGetResolver ( KService * self, const String * ticket,
+ VResolver ** resolver )
+{
+ uint32_t project = 0;
+
+ if ( self == NULL || ticket == NULL ||
+ ticket -> addr == NULL || ticket -> size == 0 || resolver == NULL)
+ {
+ return 0;
+ }
+ else {
+ const BSTItem * i = ( BSTItem * ) BSTreeFind
+ ( & self -> req . tickets . ticketsToProjects, ticket, BSTItemCmp );
+ if ( i == NULL )
+ return 0;
+
+ project = i -> project;
+ }
+
+ * resolver = NULL;
+
+ if ( project != 0 ) {
+ const KRepository * r = NULL;
+ rc_t rc = SHelperInitRepoMgr ( & self -> helper );
+ if ( rc != 0 )
+ return rc;
+
+ rc = KRepositoryMgrGetProtectedRepository
+ ( self -> helper . repoMgr, project, & r );
+ if ( rc != 0 )
+ return rc;
+
+ rc = KRepositoryMakeResolver ( r, resolver, self -> helper . kfg );
+
+ RELEASE ( KRepository, r );
+
+ return rc;
+ }
+
+ return 0;
+}
+
+static
+rc_t KServiceNamesExecuteExtImpl ( KService * self, VRemoteProtocols protocols,
+ const char * cgi, const char * version,
+ const KSrvResponse ** response, const char * expected )
+{
+ rc_t rc = 0;
+
+ KStream * stream = NULL;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ if ( response == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ if ( version == NULL )
+ version = "#3.0";
+
+ rc = KServiceInitNamesRequestWithVersion ( self, protocols, cgi, version,
+ false );
+
+ if ( rc == 0 )
+ rc = SCgiRequestPerform ( & self -> req . cgiReq, & self -> helper,
+ & stream, expected );
+
+ if ( rc == 0 )
+ rc = KServiceProcessStream ( self, stream );
+
+ if ( rc == 0 )
+ rc = KServiceGetResponse ( self, response );
+
+ RELEASE ( KStream, stream );
+
+ return rc;
+}
+
+
+/* Emulate Names Service Call :
+ - prepare the request;
+ - use "expected" instead of calling "cgi"
+ - parse "expected" as "cgi" response */
+rc_t KServiceTestNamesExecuteExt ( KService * self, VRemoteProtocols protocols,
+ const char * cgi, const char * version,
+ const struct KSrvResponse ** response, const char * expected )
+{
+ return KServiceNamesExecuteExtImpl ( self, protocols, cgi, version,
+ response, expected );
+}
+
+
+/* Execute Names Service Call : extended version */
+rc_t KServiceNamesExecuteExt ( KService * self, VRemoteProtocols protocols,
+ const char * cgi, const char * version,
+ const KSrvResponse ** response )
+{
+ return KServiceNamesExecuteExtImpl ( self, protocols, cgi, version,
+ response, NULL );
+}
+
+
+/* Execute Names Service Call using current default protocol version;
+ get KSrvResponse */
+rc_t KServiceNamesExecute ( KService * self, VRemoteProtocols protocols,
+ const KSrvResponse ** response )
+{
+ return KServiceNamesExecuteExt ( self, protocols, NULL, NULL, response );
+}
+
+
+static rc_t CC KService1NameWithVersionAndType ( const KNSManager * mgr,
+ const char * url, const char * acc, size_t acc_sz, const char * ticket,
+ VRemoteProtocols protocols, const VPath ** remote, const VPath ** mapping,
+ bool refseq_ctx, const char * version, EObjectType objectType,
+ bool aProtected )
+{
+ rc_t rc = 0;
+
+ KStream * stream = NULL;
+
+ KService service;
+
+ if ( acc == NULL || remote == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ rc = KServiceInitNames1 ( & service, mgr, url, version,
+ acc, acc_sz, ticket, protocols, objectType, refseq_ctx, aProtected );
+
+ protocols = service . req . protocols;
+
+ if ( rc == 0 )
+ rc = SCgiRequestPerform ( & service . req . cgiReq, & service . helper,
+ & stream, NULL );
+
+ if ( rc == 0 )
+ rc = KServiceProcessStream ( & service, stream );
+
+ if ( rc == 0 ) {
+ if ( VectorLength ( & service . resp . rows ) != 1)
+ rc = 1;
+ else {
+ uint32_t l = KSrvResponseLength ( service . resp . list );
+ if ( rc == 0 ) {
+ if ( l != 1 )
+ rc = 3;
+ else {
+ const KSrvError * error = NULL;
+ rc = KSrvResponseGetPath ( service . resp . list, 0,
+ protocols, NULL, NULL, & error );
+ if ( rc == 0 && error != NULL ) {
+ KSrvErrorRc ( error, & rc );
+ KSrvErrorRelease ( error );
+ }
+ else {
+ const SRow * r =
+ ( SRow * ) VectorGet ( & service . resp . rows, 0 );
+ if ( r == NULL)
+ rc = 2;
+ else {
+ const VPath * path = NULL;
+ VRemoteProtocols protos = protocols;
+ int i = 0;
+ for ( i = 0; protos != 0 && i < eProtocolMaxPref;
+ protos >>= 3, ++ i )
+ {
+ switch ( protos & eProtocolMask ) {
+ case eProtocolHttp:
+ path = r -> path . http;
+ break;
+ case eProtocolFasp:
+ path = r -> path . fasp;
+ break;
+ case eProtocolHttps:
+ path = r -> path . https;
+ break;
+ }
+ if ( path != NULL )
+ break;
+ }
+
+ /* in early versions of protocol only http path was initialized */
+ if ( path == NULL )
+ path = r -> path . http;
+
+ rc = VPathAddRef ( path );
+ if ( rc == 0 )
+ * remote = path;
+ if ( mapping ) {
+ path = r -> path . mapping;
+ rc = VPathAddRef ( path );
+ if ( rc == 0 )
+ * mapping = path;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if ( rc == 0 ) {
+ uint32_t l = KSrvResponseLength ( service . resp . list );
+ if ( l != 1)
+ rc = 3;
+ else {
+ const VPathSet * s = NULL;
+ rc = KSrvResponseGet ( service . resp . list, 0, & s );
+ if ( rc != 0 ) {
+ }
+ else if ( s == NULL )
+ rc = 4;
+ else {
+ const VPath * path = NULL;
+ const VPath * cache = NULL;
+ rc = VPathSetGet ( s, protocols, & path, & cache );
+ if ( rc == 0 ) {
+ int notequal = ~ 0;
+ assert ( remote );
+ rc = VPathEqual ( * remote, path, & notequal );
+ if ( rc == 0 )
+ rc = notequal;
+ RELEASE ( VPath, cache );
+ RELEASE ( VPath, path );
+ }
+ }
+ RELEASE ( VPathSet, s );
+ }
+ }
+
+ {
+ rc_t r2 = KServiceFini ( & service );
+ if ( rc == 0 )
+ rc = r2;
+ }
+
+ RELEASE ( KStream, stream );
+
+ return rc;
+}
+
+
+/* make name service call : request: 1 object, response: 1 object */
+LIB_EXPORT
+rc_t CC KService1NameWithVersion ( const KNSManager * mgr, const char * url,
+ const char * acc, size_t acc_sz, const char * ticket,
+ VRemoteProtocols protocols, const VPath ** remote, const VPath ** mapping,
+ bool refseq_ctx, const char * version, bool aProtected )
+{
+ if ( version == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ return KService1NameWithVersionAndType ( mgr, url, acc, acc_sz, ticket,
+ protocols, remote, mapping, refseq_ctx, version, eOT_undefined,
+ aProtected );
+}
+
+
+/* Execute Search Service Call : extended version */
+rc_t KServiceSearchExecuteExt ( KService * self, const char * cgi,
+ const char * version, const Kart ** result )
+{
+ rc_t rc = 0;
+
+ KStream * stream = NULL;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ if ( result == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ if ( version == NULL )
+ version = "#1.0";
+
+ rc = KServiceInitSearchRequestWithVersion ( self, cgi, version );
+
+ if ( rc == 0 )
+ rc = SCgiRequestPerform ( & self -> req . cgiReq, & self -> helper,
+ & stream, NULL );
+
+ if ( rc == 0 )
+ rc = KServiceProcessStream ( self, stream );
+
+ if ( rc == 0 ) {
+ rc = KartAddRef ( self -> resp . kart );
+ if ( rc == 0 )
+ * result = self -> resp . kart;
+ }
+
+ RELEASE ( KStream, stream );
+
+ return rc;
+}
+
+
+/* Execute Search Service Call; get Kart response */
+rc_t KServiceSearchExecute ( KService * self, const Kart ** response ) {
+ return KServiceSearchExecuteExt ( self, NULL, NULL, response );
+}
+
+
+/* Execute a simple ( one accession in request ) Search Service Call;
+ get Kart response */
+rc_t KService1Search ( const KNSManager * mgr, const char * cgi,
+ const char * acc, const Kart ** result )
+{
+ rc_t rc = 0;
+
+ KService service;
+
+ rc = KServiceInit ( & service, mgr );
+
+ if ( rc == 0 )
+ rc = KServiceAddId ( & service, acc );
+
+ if ( rc == 0 )
+ rc = KServiceSearchExecute ( & service, result );
+
+ {
+ rc_t r2 = KServiceFini ( & service );
+ if ( rc == 0 )
+ rc = r2;
+ }
+ return rc;
+}
+
+
+/* TESTS **********************************************************************/
+typedef struct {
+ rc_t passed;
+ const char * acc;
+ const char * version;
+ VRemoteProtocols protocols;
+} SKVCheck;
+
+static void SCgiRequestCheck ( void * item, void * data ) {
+ /* const SKV * kv = ( SKV * ) item; */
+ SKVCheck * p = ( SKVCheck * ) data;
+ assert ( p );
+ p -> passed = 0;
+}
+
+static void SKVCheckInit ( SKVCheck * self, const char * acc,
+ const char * version, VRemoteProtocols protocols )
+{
+ assert ( self );
+ memset ( self, 0, sizeof * self );
+ self -> acc = acc;
+ self -> version = version;
+ self -> protocols = protocols;
+ self -> passed = -1;
+}
+
+rc_t KServiceRequestTestNames1 ( const KNSManager * mgr,
+ const char * cgi, const char * version, const char * acc, size_t acc_sz,
+ const char * ticket, VRemoteProtocols protocols,
+ EObjectType objectType )
+{
+ KService service;
+ rc_t rc = KServiceInitNames1 ( & service, mgr, cgi, version,
+ acc, acc_sz, ticket, protocols, objectType, false, false );
+ if ( rc == 0 ) {
+ SKVCheck c;
+ SKVCheckInit ( & c, acc, version, protocols );
+ VectorForEach
+ ( & service . req . cgiReq . params, false, SCgiRequestCheck, & c );
+ rc = c . passed;
+ }
+ {
+ rc_t r2 = KServiceFini ( & service );
+ if ( rc == 0 ) {
+ rc = r2;
+ }
+ }
+ return rc;
+}
+
+rc_t KServiceNamesRequestTest ( const KNSManager * mgr, const char * b,
+ const char * cgi, VRemoteProtocols protocols,
+ const SServiceRequestTestData * d, ... )
+{
+ va_list args;
+ KService * service = NULL;
+ KStream * stream = NULL;
+ rc_t rc = KServiceMakeWithMgr ( & service, mgr);
+ va_start ( args, d );
+ while ( rc == 0 && d != NULL ) {
+ if ( d -> id != NULL ) {
+ rc = KServiceAddObject ( service, d -> id, 0, d -> type );
+ }
+ if ( rc == 0 && d -> ticket != NULL ) {
+ rc = KServiceAddTicket ( service, d -> ticket );
+ }
+ d = va_arg ( args, const SServiceRequestTestData * );
+ }
+ if ( rc == 0 ) {
+ rc = KServiceInitNamesRequest ( service, protocols, cgi );
+ }
+ if ( rc == 0 ) {
+ SKVCheck c;
+ /*SKVCheckInit ( & c, acc, service -> req . version .raw . s, protocols );*/
+ VectorForEach (
+ & service -> req . cgiReq . params, false, SCgiRequestCheck, & c );
+ rc = c . passed;
+ }
+ if ( rc == 0 ) {
+ rc = KStreamMakeFromBuffer ( & stream, b, string_size ( b ) );
+ }
+ if ( rc == 0 ) {
+ rc = KServiceProcessStream ( service, stream );
+ }
+ if ( rc == 0 ) {
+ const KSrvResponse * l = NULL;
+ rc = KServiceGetResponse ( service, & l );
+ if ( rc == 0 ) {
+ uint32_t i = 0;
+ uint32_t n = KSrvResponseLength ( l );
+ for ( i = 0; rc == 0 && i < n; ++i ) {
+ const VPathSet * s = NULL;
+ rc = KSrvResponseGet ( l, i, & s );
+ RELEASE ( VPathSet, s );
+ }
+ }
+ RELEASE ( KSrvResponse, l );
+ }
+ RELEASE ( KStream, stream );
+ RELEASE ( KService, service );
+ return rc;
+}
+
+rc_t KServiceFuserTest ( const KNSManager * mgr, const char * ticket,
+ const char * acc, ... )
+{
+ va_list args;
+ KService * service = NULL;
+ const KSrvResponse * response = NULL;
+ rc_t rc = KServiceMake ( & service);
+ va_start ( args, acc );
+ while ( rc == 0 && acc != NULL ) {
+ rc = KServiceAddId ( service, acc );
+ acc = va_arg ( args, const char * );
+ }
+ if ( rc == 0 ) {
+ rc = KServiceNamesQuery ( service, eProtocolDefault, & response );
+ }
+ if ( rc == 0 ) {
+ uint32_t i = 0;
+ for ( i = 0; rc == 0 && i < KSrvResponseLength ( response ); ++i ) {
+ const VPath * path = NULL;
+ rc = KSrvResponseGetPath ( response, i, 0, & path, NULL, NULL );
+ if ( rc == 0 ) {
+ rc_t r2;
+/*KTime_t mod = VPathGetModDate ( path );size_t size = VPathGetSize ( path );*/
+ String id;
+ memset ( & id, 0, sizeof id );
+ r2 = VPathGetId ( path, & id );
+ if ( rc == 0 )
+ rc = r2;
+ }
+ RELEASE ( VPath, path );
+ }
+ }
+ RELEASE ( KSrvResponse, response );
+ RELEASE ( KService, service );
+ return rc;
+}
+
+rc_t SCgiRequestPerformTestNames1 ( const KNSManager * mgr, const char * cgi,
+ const char * version, const char * acc, const char * ticket,
+ VRemoteProtocols protocols, EObjectType objectType )
+{
+ KService service;
+
+ rc_t rc = KServiceInitNames1 ( & service, mgr, cgi, version, acc,
+ string_measure ( acc, NULL ), ticket, protocols, objectType, false,
+ false );
+
+ if ( rc == 0 ) {
+ KStream * response = NULL;
+ rc = SCgiRequestPerform ( & service . req . cgiReq, & service . helper,
+ & response, NULL );
+ RELEASE ( KStream, response );
+ }
+
+ {
+ rc_t r2 = KServiceFini ( & service );
+ if ( rc == 0 )
+ rc = r2;
+ }
+
+ return rc;
+}
+
+rc_t KServiceProcessStreamTestNames1 ( const KNSManager * mgr,
+ const char * b, const char * version, const char * acc,
+ const VPath * exp, const char * ticket, const VPath * ex2,
+ int errors )
+{
+ KService service;
+ KStream * stream = NULL;
+ rc_t rc = 0;
+ if ( rc == 0 )
+ rc = KServiceInitNames1 ( & service, mgr, "", version, acc,
+ string_measure ( acc, NULL ), ticket, eProtocolHttps,
+ eOT_undefined, false, false );
+ if ( rc == 0 ) {
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS_SERVICE ), (
+ "XXXXXXXXXXXX NOT sending HTTP POST request XXXXXXXXXXXXXXXX\n" ) );
+ rc = KStreamMakeFromBuffer ( & stream, b, string_size ( b ) );
+ }
+ if ( rc == 0 )
+ KServiceExpectErrors ( & service, errors );
+ if ( rc == 0 )
+ rc = KServiceProcessStream ( & service, stream );
+ if ( rc == 0 ) {
+ if ( VectorLength ( & service . resp . rows ) != 1 )
+ rc = 1;
+ else {
+ const VPath * path = NULL;
+ const SRow * r
+ = ( SRow * ) VectorGet ( & service . resp . rows, 0 );
+ if ( r == NULL)
+ rc = 2;
+ else
+ if ( r -> path . error != NULL )
+ rc = r -> path . error -> rc;
+ else
+ path = r -> path . http;
+ if ( exp != NULL && rc == 0 ) {
+ int notequal = ~ 0;
+ rc = VPathEqual ( path, exp, & notequal );
+ if ( rc == 0 )
+ rc = notequal;
+ }
+ if ( ex2 != NULL && rc == 0 ) {
+ int notequal = ~ 0;
+ rc = VPathEqual ( path, ex2, & notequal );
+ if ( rc == 0 )
+ rc = notequal;
+ }
+ }
+ }
+ {
+ rc_t r2 = KServiceFini ( & service );
+ if ( rc == 0 ) {
+ rc = r2;
+ }
+ }
+ RELEASE ( KStream, stream );
+ return rc;
+}
+
+
+/* Parse "buffer" as names-3.0 response.
+ Do not log "errorsToIgnore" messages during response processing */
+rc_t KServiceNames3_0StreamTest ( const char * buffer,
+ const KSrvResponse ** response, int errorsToIgnore )
+{
+ rc_t rc = 0;
+ rc_t r2 = 0;
+
+ KStream * stream = NULL;
+
+ KService service;
+ if ( rc == 0 )
+ rc = KServiceInit ( & service, NULL );
+ if ( rc == 0 )
+ KServiceExpectErrors ( & service, errorsToIgnore );
+
+ if ( rc == 0 )
+ rc = KStreamMakeFromBuffer ( & stream, buffer, string_size ( buffer ) );
+
+ if ( rc == 0 )
+ rc = KServiceProcessStream ( & service, stream );
+
+ if ( rc == 0 )
+ rc = KServiceGetResponse ( & service , response );
+
+ r2 = KServiceFini ( & service );
+ if ( rc == 0 )
+ rc = r2;
+
+ RELEASE ( KStream, stream );
+
+ return rc;
+}
+
+rc_t KServiceCgiTest1 ( const KNSManager * mgr, const char * cgi,
+ const char * version, const char * acc, const char * ticket,
+ VRemoteProtocols protocols, EObjectType objectType,
+ const VPath * exp, const VPath * ex2 )
+{
+ const VPath * path = NULL;
+ rc_t rc = KService1NameWithVersionAndType ( mgr, cgi, acc,
+ string_measure ( acc, NULL ), ticket, protocols,
+ & path, NULL, false, version, objectType, false );
+ if ( rc == 0 ) {
+ if ( exp != NULL && rc == 0 ) {
+ int notequal = ~ 0;
+ rc = VPathEqual ( path, exp, & notequal );
+ if ( rc == 0 ) {
+ rc = notequal;
+ }
+ }
+ if ( ex2 != NULL && rc == 0 ) {
+ int notequal = ~ 0;
+ rc = VPathEqual ( path, ex2, & notequal );
+ if ( rc == 0 ) {
+ rc = notequal;
+ }
+ }
+ }
+ RELEASE ( VPath, path );
+ return rc;
+}
+
+rc_t KServiceSearchTest1
+ ( const KNSManager * mgr, const char * cgi, const char * acc )
+{
+ rc_t rc = 0;
+ KService service;
+ const Kart * result = NULL;
+ rc = KServiceInit ( & service, mgr );
+ if ( rc == 0 ) {
+ rc = KServiceAddId ( & service, acc );
+ }
+ if ( rc == 0 )
+ rc = KServiceSearchExecute ( & service, & result );
+ {
+ rc_t r2 = KServiceFini ( & service );
+ if ( rc == 0 )
+ rc = r2;
+ }
+ RELEASE ( Kart, result );
+ return rc;
+}
+
+rc_t KServiceSearchTest (
+ const KNSManager * mgr, const char * cgi, const char * acc, ... )
+{
+ va_list args;
+ rc_t rc = 0;
+ KStream * stream = NULL;
+ const Kart * result = NULL;
+ KService service;
+ rc = KServiceInit ( & service, mgr );
+ va_start ( args, acc );
+ while ( rc == 0 && acc != NULL ) {
+ rc = KServiceAddObject ( & service, acc, 0, eOT_undefined);
+ acc = va_arg ( args, const char * );
+ }
+ if ( rc == 0 ) {
+ rc = KServiceSearchExecuteExt ( & service, cgi, NULL, & result );
+ }
+ {
+ rc_t r2 = KartRelease ( result );
+ if ( rc == 0 ) {
+ rc = r2;
+ }
+ }
+ {
+ rc_t r2 = KServiceFini ( & service );
+ if ( rc == 0 ) {
+ rc = r2;
+ }
+ }
+ RELEASE ( KStream, stream );
+ return rc;
+}
+/******************************************************************************/
diff --git a/libs/vfs/resolver-priv.h b/libs/vfs/resolver-priv.h
index c82a135..2f8ab8f 100644
--- a/libs/vfs/resolver-priv.h
+++ b/libs/vfs/resolver-priv.h
@@ -35,6 +35,7 @@
extern "C" {
#endif
+#define DEFAULT_PROTOCOLS eProtocolHttpHttps
/*--------------------------------------------------------------------------
* KConfig Repository Structure
@@ -140,8 +141,6 @@ struct KNSManager;
struct String;
struct VResolverAlg;
-void VFSManagerSetNameResolverVersion3_0(void);
-
rc_t VPathCheckFromNamesCGI(const struct VPath *path,
const struct String *ticket, const struct VPath **mapping);
@@ -215,7 +214,7 @@ rc_t VResolverAlgParseResolverCGIResponse_3_0(const char *start,
rc_t VResolverAlgRemoteProtectedResolve( const struct VResolverAlg *self,
const struct KNSManager *kns, VRemoteProtocols protocols,
const struct String *acc, const struct VPath **path,
- const struct VPath **mapping, bool legacy_wgs_refseq);
+ const struct VPath **mapping, bool legacy_wgs_refseq, const char * version);
/** get projectId ( valid for protected user repository ) */
rc_t VResolverGetProjectId ( const VResolver * self, uint32_t * projectId );
@@ -232,7 +231,7 @@ rc_t VResolverGetProjectId ( const VResolver * self, uint32_t * projectId );
rc_t VResolverRemoteResolve ( const VResolver *self,
VRemoteProtocols protocols, const struct String * accession,
const struct VPath ** path, const struct VPath **mapping,
- const struct KFile ** opt_file_rtn, bool refseq_ctx, bool is_oid );
+ const struct KFile ** opt_file_rtn, bool refseq_ctx, bool is_oid, const char * version );
void KConfigReadRemoteProtocols ( struct KConfig const * self, VRemoteProtocols * remote_protos );
diff --git a/libs/vfs/resolver.c b/libs/vfs/resolver.c
index a157ea3..3e49077 100644
--- a/libs/vfs/resolver.c
+++ b/libs/vfs/resolver.c
@@ -26,6 +26,11 @@
#include <vfs/extern.h>
+
+#include <klib/time.h> /* KTime */
+
+#include "services-priv.h"
+#include "path-priv.h"
#include "resolver-priv.h"
#include <vfs/manager.h>
@@ -58,7 +63,6 @@
#include <sysalloc.h>
#include <vfs/path-priv.h>
-#include "path-priv.h"
#include <stdlib.h>
#include <string.h>
@@ -66,6 +70,9 @@
#include <os-native.h>
#include <assert.h>
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
+
/* to turn off CGI name resolution for
any refseq accessions */
#define NO_REFSEQ_CGI 0
@@ -89,14 +96,6 @@ static uint32_t NAME_SERVICE_MIN_VERS = NAME_SERVICE_MIN_VERS_;
static uint32_t NAME_SERVICE_VERS
= NAME_SERVICE_MAJ_VERS_ << 24 | NAME_SERVICE_MIN_VERS_ << 16;
-static void VFSManagerSetNameResolverVersion(uint32_t maj, uint32_t min) {
- NAME_SERVICE_MAJ_VERS = maj;
- NAME_SERVICE_MIN_VERS = min;
- NAME_SERVICE_VERS
- = NAME_SERVICE_MAJ_VERS_ << 24 | NAME_SERVICE_MIN_VERS_ << 16;
-}
-void VFSManagerSetNameResolverVersion3_0(void)
-{ VFSManagerSetNameResolverVersion(3, 0); }
/*--------------------------------------------------------------------------
* String
@@ -931,6 +930,23 @@ rc_t VResolverAlgParseResolverCGIResponse_1_0 ( const char *start, size_t size,
return rc;
}
+static int getDigit ( char c, rc_t * rc ) {
+ assert ( rc );
+
+ if ( * rc != 0 )
+ return 0;
+
+ c = tolower ( c );
+ if ( ! isdigit ( c ) && c < 'a' && c > 'f' ) {
+ * rc = RC ( rcVFS, rcQuery, rcExecuting, rcItem, rcIncorrect );
+ return 0;
+ }
+
+ if ( isdigit ( c ) )
+ return c - '0';
+
+ return c - 'a' + 10;
+}
/* ParseResolverCGIResponse_1_1
* expect single row table, with this structure (SRA-1690) :
@@ -943,7 +959,7 @@ rc_t VResolverAlgParseResolverCGIResponse_1_1 ( const char *astart, size_t size,
const String *ticket )
{
const char *start = astart;
- rc_t rc;
+ rc_t rc = 0;
KLogLevel lvl;
char *rslt_end;
uint32_t result_code;
@@ -1070,20 +1086,48 @@ rc_t VResolverAlgParseResolverCGIResponse_1_1 ( const char *astart, size_t size,
but can only handle 200 */
if ( result_code == 200 )
{
- /* normal public response */
+ uint8_t ud5 [ 16 ];
+ bool has_md5 = false;
+ KTime_t date = 0;
+ size_t size = 0;
+ if ( size_str . size != 0 && size_str . len != 0 ) {
+ rc_t r2 = 0;
+ size = StringToU64 ( & size_str, & r2 );
+ if ( r2 != 0 )
+ size = 0;
+ }
+ if ( mod_date . addr != NULL && mod_date . size > 0 ) {
+ KTime kt;
+ const KTime * t = KTimeFromIso8601 ( & kt, mod_date . addr,
+ mod_date . size );
+ if ( t != NULL )
+ date = KTimeMakeTime ( & kt );
+ }
+ if ( md5 . addr != NULL && md5 . size == 32 ) {
+ int i = 0;
+ for ( i = 0; i < 16 && rc == 0; ++ i ) {
+ ud5 [ i ] = getDigit ( md5 . addr [ 2 * i ], & rc ) * 16;
+ ud5 [ i ] += getDigit ( md5 . addr [ 2 * i + 1 ], & rc );
+ }
+ has_md5 = rc == 0;
+ }
+ /* normal public response *
if ( download_ticket . size == 0
#if DO_NOT_USE_TIC_HACK
|| mapping != NULL
#endif
)
- {
- rc = VPathMakeFmt ( ( VPath** ) path, "%S", & url );
- }
+ {*/
+ rc = VPathMakeFromUrl ( ( VPath** ) path, & url,
+ & download_ticket, true, & accession, size, date,
+ has_md5 ? ud5 : NULL, 0 );
+ /*}
else
{
- /* protected response */
- rc = VPathMakeFmt ( ( VPath** ) path, "%S?tic=%S", & url, & download_ticket );
- }
+ * protected response *
+ rc = VPath MakeFmtExt ( ( VPath** ) path, true, & accession,
+ size, date, "%S?tic=%S", & url, & download_ticket );
+ }*/
if ( rc == 0 )
{
@@ -1266,7 +1310,8 @@ rc_t VResolverAlgParseResolverCGIResponse ( const KDataBuffer *result,
start = ( const void* ) result -> base;
size = KDataBufferBytes ( result );
- DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), (" Response = %.*s\n", size, start));
+ DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
+ (" Response = %.*s\n", ( int ) size, start));
/* peel back buffer to significant bytes */
while ( size > 0 && start [ size - 1 ] == 0 ) -- size;
@@ -1328,11 +1373,12 @@ static uint32_t TESTING_VDB_3162_CODE ( rc_t rc, uint32_t code ) {
}
#endif
-
+#ifdef TESTING_SERVICES_VS_OLD_RESOLVING
/* RemoteProtectedResolve
* use NCBI CGI to resolve accession into URL
*/
-rc_t VResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
+static
+rc_t oldVResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
const KNSManager *kns, VRemoteProtocols protocols, const String *acc,
const VPath ** path, const VPath ** mapping, bool legacy_wgs_refseq )
{
@@ -1487,7 +1533,7 @@ rc_t VResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
KStreamRelease ( response );
}
}
- else if ( code == 403 ) {
+ else if ( code == 403 ) { // TODO CHECK AGAINS SERVICES
/* HTTP/1.1 403 Forbidden
- resolver CGI was called over http insted of https */
rc = RC ( rcVFS, rcResolver, rcResolving,
@@ -1518,6 +1564,34 @@ rc_t VResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
return rc;
}
+#endif
+
+rc_t VResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
+ const KNSManager *kns, VRemoteProtocols protocols, const String *acc,
+ const VPath ** path, const VPath ** mapping, bool legacy_wgs_refseq,
+ const char * version )
+{
+ rc_t rc = 0;
+ const char * ticket = NULL;
+
+ assert ( self && self -> root && acc );
+ if ( self -> ticket != NULL ) {
+ ticket = self -> ticket -> addr;
+ }
+
+ rc = KService1NameWithVersion ( kns, self -> root -> addr,
+ acc -> addr, acc -> len, ticket, protocols, path, mapping,
+ legacy_wgs_refseq, version, self -> protected );
+
+ assert(*path != NULL || rc != 0);
+
+ if (rc == 0 && *path == NULL)
+ {
+ rc = RC(rcVFS, rcResolver, rcResolving, rcName, rcNull);
+ }
+
+ return rc;
+}
/* If resolver-cgi is on government site and is called over http:
fix it to https */
@@ -1610,7 +1684,8 @@ rc_t VResolverAlgFixHTTPSOnlyStandard ( VResolverAlg * self, bool * fixed )
static
rc_t VResolverAlgRemoteResolve ( const VResolverAlg *self,
const KNSManager *kns, VRemoteProtocols protocols, const VResolverAccToken *tok,
- const VPath ** path, const VPath ** mapping, const KFile ** opt_file_rtn, bool legacy_wgs_refseq )
+ const VPath ** path, const VPath ** mapping, const KFile ** opt_file_rtn, bool legacy_wgs_refseq,
+ const char * version )
{
rc_t rc;
uint32_t i, count;
@@ -1634,10 +1709,21 @@ rc_t VResolverAlgRemoteResolve ( const VResolverAlg *self,
bool done = false;
int i = 0;
for ( i = 0; i < 2 && ! done; ++i ) {
- rc = VResolverAlgRemoteProtectedResolve ( self, kns,
+ if ( version == NULL )
+#ifdef TESTING_SERVICES_VS_OLD_RESOLVING
+ rc = oldVResolverAlgRemoteProtectedResolve ( self, kns,
protocols, & tok -> acc, path, mapping, legacy_wgs_refseq );
+#else
+ assert(0);
+#endif
+ else
+ rc = VResolverAlgRemoteProtectedResolve ( self, kns,
+ protocols, & tok -> acc, path, mapping, legacy_wgs_refseq, version );
if ( rc == SILENT_RC (
- rcVFS, rcResolver, rcResolving, rcConnection, rcUnauthorized ) )
+ rcVFS, rcResolver, rcResolving, rcConnection, rcUnauthorized )
+ || rc == SILENT_RC (
+ rcVFS, rcQuery, rcExecuting, rcConnection, rcUnauthorized )
+ )
{ /* resolver-cgi is called over http instead of https:
fix it */
bool fixed = false;
@@ -2292,14 +2378,14 @@ rc_t VResolverFuseMountedResolve ( const VResolver * self,
PathType = KDirectoryPathType (
NativeDir,
"%.*s",
- accession -> size,
+ ( int ) accession -> size,
accession -> addr
);
if ( PathType == kptFile ) {
rc = VPathMakeFmt (
( VPath ** ) path,
"%.*s",
- accession -> size,
+ ( int ) accession -> size,
accession -> addr
);
}
@@ -2573,7 +2659,7 @@ VResolverEnableState CC VResolverCacheEnable ( const VResolver * self, VResolver
rc_t VResolverRemoteResolve ( const VResolver *self,
VRemoteProtocols protocols, const String * accession,
const VPath ** path, const VPath **mapping,
- const KFile ** opt_file_rtn, bool refseq_ctx, bool is_oid )
+ const KFile ** opt_file_rtn, bool refseq_ctx, bool is_oid, const char * version )
{
rc_t rc, try_rc;
uint32_t i, count;
@@ -2620,7 +2706,8 @@ rc_t VResolverRemoteResolve ( const VResolver *self,
const VResolverAlg *alg = VectorGet ( & self -> remote, i );
if ( alg -> app_id == app || alg -> app_id == wildCard )
{
- try_rc = VResolverAlgRemoteResolve ( alg, self -> kns, protocols, & tok, path, mapping, opt_file_rtn, legacy_wgs_refseq );
+ try_rc = VResolverAlgRemoteResolve ( alg, self -> kns, protocols,
+ & tok, path, mapping, opt_file_rtn, legacy_wgs_refseq, version );
if ( try_rc == 0 )
return 0;
if ( rc == 0 )
@@ -2635,7 +2722,8 @@ rc_t VResolverRemoteResolve ( const VResolver *self,
const VResolverAlg *alg = VectorGet ( & self -> remote, i );
if ( ( alg -> app_id == app || alg -> app_id == wildCard ) && ! alg -> disabled )
{
- try_rc = VResolverAlgRemoteResolve ( alg, self -> kns, protocols, & tok, path, mapping, opt_file_rtn, legacy_wgs_refseq );
+ try_rc = VResolverAlgRemoteResolve ( alg, self -> kns, protocols,
+ & tok, path, mapping, opt_file_rtn, legacy_wgs_refseq, version );
if ( try_rc == 0 )
return 0;
if ( rc == 0 )
@@ -3024,8 +3112,9 @@ rc_t VResolverQueryOID ( const VResolver * self, VRemoteProtocols protocols,
if ( rc == 0 )
{
const VPath * remote2, * remote_mapping = NULL;
- rc = VResolverRemoteResolve ( self, protocols, & accession,
- & remote2, & remote_mapping, NULL, refseq_ctx, true );
+ rc = VResolverRemoteResolve ( self, protocols,
+ & accession, & remote2, & remote_mapping, NULL,
+ refseq_ctx, true, NULL );
if ( rc == 0 )
{
/* got it. now enter into VFS manager's table */
@@ -3097,9 +3186,10 @@ rc_t VResolverQueryOID ( const VResolver * self, VRemoteProtocols protocols,
if ( rc == 0 )
{
const VPath * remote_mapping = NULL;
- rc = VResolverRemoteResolve ( self, protocols, & accession, remote,
- ( mapped_query == NULL && cache != NULL ) ? & remote_mapping : NULL,
- NULL, refseq_ctx, true );
+ rc = VResolverRemoteResolve ( self, protocols,
+ & accession, remote,
+ ( mapped_query == NULL && cache != NULL ) ? & remote_mapping : NULL,
+ NULL, refseq_ctx, true, NULL );
if ( rc == 0 && mapped_query == NULL && cache != NULL && remote_mapping == NULL )
{
@@ -3166,7 +3256,7 @@ rc_t VResolverQueryOID ( const VResolver * self, VRemoteProtocols protocols,
*/
static
rc_t VResolverQueryAcc ( const VResolver * self, VRemoteProtocols protocols,
- const VPath * query, const VPath ** local, const VPath ** remote, const VPath ** cache )
+ const VPath * query, const VPath ** local, const VPath ** remote, const VPath ** cache, const char * version )
{
rc_t rc = 0;
@@ -3197,7 +3287,7 @@ rc_t VResolverQueryAcc ( const VResolver * self, VRemoteProtocols protocols,
/* request remote resolution
this does not need to map the query to an accession */
rc = VResolverRemoteResolve ( self, protocols, accession,
- & remote2, mapped_ptr, NULL, refseq_ctx, false );
+ & remote2, mapped_ptr, NULL, refseq_ctx, false, version );
if ( rc == 0 )
{
@@ -3392,7 +3482,7 @@ rc_t VResolverQueryURL ( const VResolver * self, VRemoteProtocols protocols,
*/
static
rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
- const VPath * query, const VPath ** local, const VPath ** remote, const VPath ** cache )
+ const VPath * query, const VPath ** local, const VPath ** remote, const VPath ** cache, const char * version )
{
rc_t rc;
@@ -3400,7 +3490,7 @@ rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
rc = RC ( rcVFS, rcResolver, rcResolving, rcParam, rcNull );
else
{
- if ( protocols == 0 )
+ if ( protocols == eProtocolDefault )
protocols = self -> protocols;
if ( local != NULL )
@@ -3496,7 +3586,7 @@ rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
break;
case vpAccession:
- rc = VResolverQueryAcc ( self, protocols, query, local, remote, cache );
+ rc = VResolverQueryAcc ( self, protocols, query, local, remote, cache, version );
break;
case vpNameOrOID:
@@ -3506,7 +3596,7 @@ rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
break;
case vpNameOrAccession:
- rc = VResolverQueryAcc ( self, protocols, query, local, remote, cache );
+ rc = VResolverQueryAcc ( self, protocols, query, local, remote, cache, version );
if ( rc != 0 )
goto try_name;
break;
@@ -3517,7 +3607,7 @@ rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
{
if ( VPathHasRefseqContext ( query ) )
{
- rc = VResolverQueryAcc ( self, protocols, query, local, remote, cache );
+ rc = VResolverQueryAcc ( self, protocols, query, local, remote, cache, version );
if ( rc == 0 )
break;
}
@@ -3541,11 +3631,34 @@ rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
return rc;
}
+#ifdef TESTING_SERVICES_VS_OLD_RESOLVING
+LIB_EXPORT
+rc_t CC oldVResolverQuery ( const VResolver * self, VRemoteProtocols protocols,
+ const VPath * query, const VPath ** local, const VPath ** remote, const VPath ** cache )
+{
+ rc_t rc = VResolverQueryInt ( self, protocols, query, local, remote, cache,
+ NULL );
+ if ( rc == 0 )
+ {
+ /* the paths returned from resolver are highly reliable */
+ if ( local != NULL && * local != NULL )
+ VPathMarkHighReliability ( * ( VPath ** ) local, true );
+ if ( remote != NULL && * remote != NULL )
+ VPathMarkHighReliability ( * ( VPath ** ) remote, true );
+ if ( cache != NULL && * cache != NULL )
+ VPathMarkHighReliability ( * ( VPath ** ) cache, true );
+ }
+ return rc;
+}
+#endif
+
LIB_EXPORT
rc_t CC VResolverQuery ( const VResolver * self, VRemoteProtocols protocols,
const VPath * query, const VPath ** local, const VPath ** remote, const VPath ** cache )
{
- rc_t rc = VResolverQueryInt ( self, protocols, query, local, remote, cache );
+ rc_t rcs = -1;
+ rc_t rc = rcs = VResolverQueryInt
+ ( self, protocols, query, local, remote, cache, "#3.0" );
if ( rc == 0 )
{
/* the paths returned from resolver are highly reliable */
@@ -3556,6 +3669,120 @@ rc_t CC VResolverQuery ( const VResolver * self, VRemoteProtocols protocols,
if ( cache != NULL && * cache != NULL )
VPathMarkHighReliability ( * ( VPath ** ) cache, true );
}
+#ifdef TESTING_SERVICES_VS_OLD_RESOLVING
+ {
+ const VPath * oath = NULL;
+ const VPath ** p = remote ? & oath : NULL;
+ const VPath * cath = NULL;
+ const VPath ** c = cache ? & cath : NULL;
+ const VPath * lath = NULL;
+ const VPath ** l = local ? & lath : NULL;
+ rc_t ro = oldVResolverQuery ( self, protocols, query, l, p, c );
+ /*ros = ro;*/
+ if ( rc != ro ) {
+ enum RCModule mod = GetRCModule ( rc );
+ enum RCTarget targ = GetRCTarget ( rc );
+ enum RCContext ctx = GetRCContext ( rc );
+ if ( targ == rcQuery && GetRCTarget ( ro ) == rcTree ) {
+ rc = ResetRCContext ( rc, mod, rcTree, ctx );
+ /*fixed = true;*/
+ }
+ }
+ assert ( rc == ro );
+ if ( remote == NULL )
+ assert ( p == NULL );
+ else if ( * remote == NULL )
+ assert ( p && * p == NULL && oath == NULL );
+ else {
+ int notequal = ~ 0;
+ assert ( ! VPathEqual ( * remote, oath, & notequal ) );
+ if ( notequal ) {
+ assert ( VPathHasRefseqContext ( query ) && notequal == 2 );
+ }
+ }
+ if ( cache == NULL )
+ assert ( c == NULL );
+ else if ( * cache == NULL )
+ assert ( c && * c == NULL && cath == NULL );
+ else {
+ int notequal = ~ 0;
+ VPathMarkHighReliability ( ( VPath * ) cath, true );
+ assert ( ! VPathEqual ( * cache, cath, & notequal ) );
+ if ( notequal ) {
+ assert ( VPathHasRefseqContext ( query ) && notequal == 2 );
+ }
+ }
+ if ( local == NULL )
+ assert ( l == NULL );
+ else if ( * local == NULL )
+ assert ( l && * l == NULL && lath == NULL );
+ else {
+ VPathMarkHighReliability ( ( VPath * ) lath, true );
+#if _DEBUGGING
+ {
+ int notequal = ~ 0;
+ assert ( ! VPathEqual ( * local, lath, & notequal ) );
+ assert ( ! notequal );
+ }
+#endif
+ }
+ RELEASE ( VPath, lath );
+ RELEASE ( VPath, oath );
+ RELEASE ( VPath, cath );
+ }
+ {
+ const VPath * oath = NULL;
+ const VPath ** p = remote ? & oath : NULL;
+ const VPath * cath = NULL;
+ const VPath ** c = cache ? & cath : NULL;
+ const VPath * lath = NULL;
+ const VPath ** l = local ? & lath : NULL;
+#if _DEBUGGING
+ rc_t ro =
+#endif
+ VResolverQueryInt ( self, protocols, query, l, p, c, "#1.2" );
+ assert ( rcs == ro );
+ if ( remote == NULL )
+ assert ( p == NULL );
+ else if ( * remote == NULL )
+ assert ( p && * p == NULL && oath == NULL );
+ else {
+ int notequal = ~ 0;
+ assert ( ! VPathEqual ( * remote, oath, & notequal ) );
+ if ( notequal )
+ assert ( VPathHasRefseqContext ( query ) && notequal == 2 );
+ }
+ if ( cache == NULL )
+ assert ( c == NULL );
+ else if ( * cache == NULL )
+ assert ( c && * c == NULL && cath == NULL );
+ else {
+ int notequal = ~ 0;
+ VPathMarkHighReliability ( ( VPath * ) cath, true );
+ assert ( ! VPathEqual ( * cache, cath, & notequal ) );
+ if ( notequal )
+ assert ( VPathHasRefseqContext ( query ) && notequal == 2 );
+ }
+ if ( local == NULL )
+ assert ( l == NULL );
+ else if ( * local == NULL )
+ assert ( l && * l == NULL && lath == NULL );
+ else {
+ VPathMarkHighReliability ( ( VPath * ) lath, true );
+#if _DEBUGGING
+ {
+ int notequal = ~ 0;
+ assert ( ! VPathEqual ( * local, lath, & notequal ) );
+ assert ( ! notequal );
+ }
+#endif
+ }
+ RELEASE ( VPath, lath );
+ RELEASE ( VPath, oath );
+ RELEASE ( VPath, cath );
+ }
+#endif
+
return rc;
}
@@ -4686,7 +4913,7 @@ rc_t CC VResolverProtocols ( VResolver * self, VRemoteProtocols protocols )
if ( self == NULL )
return RC ( rcVFS, rcResolver, rcUpdating, rcSelf, rcNull );
- if ( protocols == 0)
+ if ( protocols == eProtocolDefault )
self -> protocols = self -> dflt_protocols;
else
{
@@ -4761,7 +4988,7 @@ rc_t VResolverMake ( VResolver ** objp, const KDirectory *wd,
/* set up protocols */
- obj -> dflt_protocols = eProtocolHttpHttps;
+ obj -> dflt_protocols = DEFAULT_PROTOCOLS;
if ( kfg != NULL )
KConfigReadRemoteProtocols ( kfg, & obj -> dflt_protocols );
diff --git a/libs/vfs/services-priv.h b/libs/vfs/services-priv.h
new file mode 100644
index 0000000..8398efe
--- /dev/null
+++ b/libs/vfs/services-priv.h
@@ -0,0 +1,126 @@
+#ifndef _h_libs_vfs_services_priv_
+#define _h_libs_vfs_services_priv_
+
+
+/*===========================================================================
+*
+* Public Domain Notice
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+
+#include <kfg/kart.h> /* EObjectType */
+#include <vfs/resolver.h> /* VRemoteProtocols */
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+struct KNSManager;
+struct KService;
+struct KSrvResponse;
+struct VPathSet;
+
+
+/* make name service call : request: 1 object, response: 1 object */
+VFS_EXTERN
+rc_t CC KService1NameWithVersion ( const struct KNSManager * mgr,
+ const char * cgi_url, const char * acc, size_t acc_sz,
+ const char * ticket, VRemoteProtocols protocols,
+ const struct VPath ** remote, const struct VPath ** mapping,
+ bool refseq_ctx, const char * names_version, bool aProtected );
+
+
+/******************************** KSrvResponse ********************************/
+rc_t KSrvResponseMake ( struct KSrvResponse ** self );
+rc_t KSrvResponseAddRef ( const struct KSrvResponse * self );
+rc_t KSrvResponseAppend ( struct KSrvResponse * self,
+ const struct VPathSet * set );
+rc_t KSrvResponseAddLocalAndCache ( struct KSrvResponse * self, uint32_t idx,
+ const struct VPathSet * localAndCache );
+rc_t KSrvResponseGet ( const struct KSrvResponse * self, uint32_t idx,
+ const struct VPathSet ** set );
+/**************************** KServiceNamesExecute ****************************/
+/* Execute Names Service Call using current default protocol version;
+ get KSrvResponse (remote-only resolution) */
+rc_t KServiceNamesExecute ( struct KService * self, VRemoteProtocols protocols,
+ const struct KSrvResponse ** response );
+/***************** Interface services.c -> remote-services.c *****************/
+rc_t KServiceGetConfig ( struct KService * self, const struct KConfig ** kfg);
+rc_t KServiceGetResolver ( struct KService * self, const String * ticket,
+ VResolver ** resolver );
+/******************************** TESTS ********************************/
+typedef struct {
+ const char * id;
+ EObjectType type;
+ const char * ticket;
+} SServiceRequestTestData;
+
+rc_t KServiceCgiTest1 ( const struct KNSManager * mgr, const char * cgi,
+ const char * version, const char * acc, const char * ticket,
+ VRemoteProtocols protocols, EObjectType objectType,
+ const struct VPath * exp, const struct VPath * ex2 );
+
+rc_t KServiceFuserTest ( const struct KNSManager * mgr, const char * ticket,
+ const char * acc, ... );
+
+rc_t SCgiRequestPerformTestNames1 ( const struct KNSManager * mgr,
+ const char * cgi, const char * version, const char * acc,
+ const char * ticket, VRemoteProtocols protocols, EObjectType objectType );
+rc_t KServiceProcessStreamTestNames1 ( const struct KNSManager * mgr,
+ const char * b, const char * version, const char * acc,
+ const struct VPath * exp, const char * ticket, const struct VPath * ex2,
+ int errors );
+rc_t KServiceRequestTestNames1 ( const struct KNSManager * mgr,
+ const char * cgi, const char * version, const char * acc, size_t acc_sz,
+ const char * ticket, VRemoteProtocols protocols,
+ EObjectType objectType );
+
+/* Parse "buffer" as names-3.0 response.
+ Do not log "errorsToIgnore" messages during response processing */
+rc_t KServiceNames3_0StreamTest ( const char * buffer,
+ const struct KSrvResponse ** response, int errorsToIgnore );
+rc_t KServiceNamesRequestTest ( const struct KNSManager * mgr, const char * b,
+ const char * cgi, VRemoteProtocols protocols,
+ const SServiceRequestTestData * d, ... );
+
+rc_t KServiceSearchTest1
+ ( const struct KNSManager * mgr, const char * cgi, const char * acc );
+rc_t KServiceSearchTest (
+ const struct KNSManager * mgr, const char * cgi, const char * acc, ... );
+/******************************************************************************/
+
+/* THE FOLLOWING DEFINE TURNS ON COMPARING OLD/NEW RESOLVING CALLS AND
+ ASSERTING WHEN THE RESULTS DO NOT MATCH.
+ REMOVE IT WHEN MERGING THE BRANCH */
+#define TESTING_SERVICES_VS_OLD_RESOLVING 1
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _h_libs_vfs_services_priv_ */
diff --git a/libs/vfs/services.c b/libs/vfs/services.c
new file mode 100644
index 0000000..f2c55ba
--- /dev/null
+++ b/libs/vfs/services.c
@@ -0,0 +1,280 @@
+/*===========================================================================
+*
+* Public Domain Notice
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+
+#include "path-priv.h" /* EVPathInitError */
+#include "services-priv.h" /* KServiceGetResolver */
+#include <kfg/config.h> /* KConfigRelease */
+#include <klib/container.h> /* BSTree */
+#include <klib/rc.h> /* RC */
+#include <vfs/manager.h> /* VFSManagerRelease */
+#include <vfs/path.h> /* VFSManagerMakePath */
+#include <vfs/services-priv.h> /* KServiceNamesExecuteExt */
+
+
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
+
+
+typedef struct {
+ BSTNode n;
+ const String * ticket;
+ const VResolver * resolver;
+} BSTItem;
+
+static void BSTItemWhack ( BSTNode * n, void * ignore ) {
+ BSTItem * i = ( BSTItem * ) n;
+
+ assert ( i );
+
+ free ( ( void * ) i -> ticket );
+ VResolverRelease ( i -> resolver );
+
+ memset ( i, 0, sizeof * i );
+}
+
+static int64_t CC BSTItemCmp ( const void * item, const BSTNode * n ) {
+ const String * s = item;
+ const BSTItem * i = ( BSTItem * ) n;
+
+ assert ( s && i );
+
+ return string_cmp ( s -> addr, s -> size,
+ i -> ticket -> addr, i -> ticket -> size, s -> size );
+}
+
+static int64_t CC BSTreeSort ( const BSTNode * item, const BSTNode * n ) {
+ const BSTItem * i = ( BSTItem * ) item;
+
+ assert ( i );
+
+ return BSTItemCmp ( i -> ticket, n );
+}
+
+
+typedef struct {
+ KService * service; /* DO NOT RELEASE */
+ VFSManager * mgr;
+ const KConfig * kfg;
+ VResolver * resolver;
+ BSTree ticketsToResolvers;
+} H;
+
+static rc_t HInit ( H * self, KService * s ) {
+ rc_t rc = 0;
+
+ assert ( self && s );
+
+ memset ( self, 0, sizeof * self );
+
+ self -> service = s;
+
+ if ( rc == 0 )
+ rc = VFSManagerMake ( & self -> mgr );
+
+ if ( rc == 0 )
+ rc = KServiceGetConfig ( s, & self -> kfg );
+
+ return rc;
+}
+
+static rc_t HFini ( H * self ) {
+ rc_t rc = 0;
+
+ assert ( self );
+
+ RELEASE ( VResolver, self -> resolver );
+ RELEASE ( KConfig, self -> kfg );
+ RELEASE ( VFSManager, self -> mgr );
+
+ BSTreeWhack ( & self -> ticketsToResolvers, BSTItemWhack, NULL );
+
+ return rc;
+}
+
+static rc_t HResolver ( H * self, const String * ticket,
+ const VResolver ** resolver )
+{
+ rc_t rc = 0;
+
+ assert ( self && resolver );
+
+ if ( ticket && ticket -> addr && ticket -> size ) {
+ BSTItem * i = ( BSTItem * ) BSTreeFind
+ ( & self -> ticketsToResolvers, ticket, BSTItemCmp );
+
+ if ( i != NULL )
+ * resolver = i -> resolver;
+ else {
+ VResolver * resolver = NULL;
+ rc = KServiceGetResolver ( self -> service, ticket, & resolver );
+ if ( rc == 0 ) {
+ i = calloc ( 1, sizeof * i );
+ if ( i == NULL )
+ return RC (
+ rcVFS, rcStorage, rcAllocating, rcMemory, rcExhausted );
+
+ rc = StringCopy ( & i -> ticket, ticket );
+ if ( rc != 0 )
+ return rc;
+
+ i -> resolver = resolver;
+ rc = BSTreeInsert ( & self -> ticketsToResolvers,
+ ( BSTNode * ) i, BSTreeSort );
+ }
+ }
+
+ assert ( i );
+
+ * resolver = i -> resolver;
+ }
+ else {
+ if ( self -> resolver == NULL )
+ rc = VFSManagerMakeResolver ( self -> mgr, & self -> resolver,
+ self -> kfg );
+
+ * resolver = self -> resolver;
+ }
+
+ return rc;
+}
+
+
+static rc_t VResolversQuery ( const VResolver * self, const VFSManager * mgr,
+ VRemoteProtocols protocols, const String * acc, VPathSet ** result )
+{
+ rc_t rc = 0;
+
+ VPath * query = NULL;
+
+ uint32_t oid = 0;
+ uint32_t i = 0;
+
+ assert ( result );
+
+ for ( i = 0; i < acc -> size; ++i ) {
+ char c = acc -> addr [ i ];
+ if ( c < '0' || c > '9' ) {
+ oid = 0;
+ break;
+ }
+ oid = oid * 10 + c - '0';
+ }
+
+ if ( oid == 0 )
+ rc = VFSManagerMakePath ( mgr, & query, "%S", acc );
+ else
+ rc = VFSManagerMakeOidPath ( mgr, & query, oid );
+
+ if ( rc == 0 ) {
+ const VPath * local = NULL;
+ const VPath * cache = NULL;
+
+ rc_t localRc = 0;
+ rc_t cacheRc = 0;
+
+ localRc = VResolverQuery ( self, protocols, query,
+ & local, NULL, NULL );
+ cacheRc = VResolverQuery ( self, protocols, query,
+ NULL, NULL, & cache );
+
+ VPathSetMakeQuery ( result, local, localRc, cache, cacheRc );
+
+ RELEASE ( VPath, local );
+ RELEASE ( VPath, cache );
+ }
+
+ RELEASE ( VPath, query );
+
+ return rc;
+}
+
+rc_t KServiceNamesQueryExt ( KService * self, VRemoteProtocols protocols,
+ const char * cgi, const char * version,
+ const KSrvResponse ** aResponse )
+{
+ rc_t rc = 0;
+ KSrvResponse * response = NULL;
+ if ( aResponse == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+ * aResponse = NULL;
+
+ {
+ const KSrvResponse * r = NULL;
+ rc = KServiceNamesExecuteExt ( self, protocols, cgi,
+ version, & r );
+ if ( rc == 0 )
+ response = ( KSrvResponse* ) r;
+ }
+
+ if ( rc == 0 ) {
+ H h;
+ rc = HInit ( & h, self );
+ {
+ uint32_t i = 0;
+ uint32_t n = KSrvResponseLength ( response );
+ for ( i = 0; rc == 0 && i < n; ++ i ) {
+ VPathSet * vps = NULL;
+ const VPath * path = NULL;
+ const KSrvError * error = NULL;
+ rc = KSrvResponseGetPath
+ ( response, i, protocols, & path, NULL, & error );
+ if ( error == NULL && rc == 0 ) {
+ const VResolver * resolver = NULL;
+ String id;
+ String ticket;
+ rc = VPathGetId ( path, & id );
+ if ( rc == 0 )
+ rc = VPathGetTicket ( path, & ticket );
+ if ( rc == 0 )
+ rc = HResolver ( & h, & ticket, & resolver );
+ if ( rc == 0 ) {
+ assert ( resolver );
+ rc = VResolversQuery ( resolver, h . mgr,
+ protocols, & id, & vps );
+ }
+ }
+ if ( vps != NULL ) {
+ rc = KSrvResponseAddLocalAndCache ( response, i, vps );
+ RELEASE ( VPathSet, vps );
+ }
+ RELEASE ( VPath, path );
+ }
+ * aResponse = response;
+ }
+ {
+ rc_t r2 = HFini ( & h );
+ if ( rc == 0 )
+ rc = r2;
+ }
+ }
+
+ return rc;
+}
+
+rc_t KServiceNamesQuery ( KService * self, VRemoteProtocols protocols,
+ const KSrvResponse ** aResponse )
+{ return KServiceNamesQueryExt ( self, protocols, NULL, NULL, aResponse ); }
diff --git a/libs/vfs/srv-response.c b/libs/vfs/srv-response.c
new file mode 100644
index 0000000..7537de3
--- /dev/null
+++ b/libs/vfs/srv-response.c
@@ -0,0 +1,580 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include "resolver-priv.h" /* DEFAULT_PROTOCOLS */
+#include "path-priv.h" /* VPathGetScheme_t */
+#include <vfs/services.h> /* KSrvResponse */
+#include <klib/rc.h> /* RC */
+#include <klib/vector.h> /* Vector */
+
+
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 && !rc) { rc = rc2; } obj = NULL; } while (false)
+
+struct VPathSet {
+ atomic32_t refcount;
+
+ const VPath * fasp;
+ const VPath * file;
+ const VPath * http;
+ const VPath * https;
+ const VPath * s3;
+ const VPath * cacheFasp;
+ const VPath * cacheFile;
+ const VPath * cacheHttp;
+ const VPath * cacheHttps;
+ const VPath * cacheS3;
+
+ const struct KSrvError * error;
+
+ const VPath * local;
+ const VPath * cache;
+ /* rc code after call to VResolverQuery(&local,&cache) */
+ rc_t localRc;
+ rc_t cacheRc;
+};
+
+struct KSrvResponse {
+ atomic32_t refcount;
+
+ Vector list;
+};
+
+/* VPathSet */
+rc_t VPathSetAddRef ( const VPathSet * self ) {
+ if ( self != NULL )
+ atomic32_inc ( & ( ( VPathSet * ) self ) -> refcount );
+
+ return 0;
+}
+
+rc_t VPathSetWhack ( VPathSet * self ) {
+ rc_t rc = 0;
+
+ if ( self != NULL ) {
+ RELEASE ( VPath, self -> fasp );
+ RELEASE ( VPath, self -> file );
+ RELEASE ( VPath, self -> http );
+ RELEASE ( VPath, self -> https );
+ RELEASE ( VPath, self -> s3 );
+ RELEASE ( VPath, self -> cacheFasp );
+ RELEASE ( VPath, self -> cacheFile );
+ RELEASE ( VPath, self -> cacheHttp );
+ RELEASE ( VPath, self -> cacheHttps );
+ RELEASE ( VPath, self -> cacheS3 );
+
+ RELEASE ( KSrvError, self -> error );
+
+ free ( self );
+ }
+
+ return rc;
+}
+
+static void whackVPathSet ( void * self, void * ignore ) {
+ VPathSetWhack ( ( VPathSet * ) self);
+}
+
+rc_t VPathSetRelease ( const VPathSet * cself ) {
+ VPathSet * self = ( VPathSet * ) cself;
+
+ if ( self != NULL && atomic32_dec_and_test ( & self -> refcount ) )
+ return VPathSetWhack ( self );
+
+ return 0;
+}
+
+rc_t VPathSetGet ( const VPathSet * self, VRemoteProtocols protocols,
+ const VPath ** path, const VPath ** vdbcache )
+{
+ rc_t rc = 0;
+ VRemoteProtocols protocol = protocols;
+ const VPath * p = NULL;
+ const VPath * c = NULL;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+ if ( protocols == eProtocolDefault )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcInvalid );
+ if ( self -> error != NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcError, rcExists );
+
+ for ( ; protocol != 0; protocol >>= 3 ) {
+ switch ( protocol & eProtocolMask ) {
+ case eProtocolFasp:
+ p = self -> fasp;
+ c = self -> cacheFasp;
+ break;
+ case eProtocolFile:
+ p = self -> file;
+ c = self -> cacheFile;
+ break;
+ case eProtocolHttp:
+ p = self -> http;
+ c = self -> cacheHttp;
+ break;
+ case eProtocolHttps:
+ p = self -> https;
+ c = self -> cacheHttps;
+ break;
+ case eProtocolS3:
+ p = self -> s3;
+ c = self -> cacheS3;
+ break;
+ default:
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcInvalid );
+ }
+
+ if ( p != NULL || c != NULL ) {
+ if ( path != NULL ) {
+ rc = VPathAddRef ( p );
+ if ( rc == 0 )
+ * path = p;
+ }
+
+ if ( vdbcache != NULL ) {
+ rc_t r2 = VPathAddRef ( c );
+ if ( r2 == 0 )
+ * vdbcache = c;
+ else if ( rc == 0)
+ rc = r2;
+ }
+
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+static
+rc_t VPathSetGetLocal ( const VPathSet * self, const VPath ** path )
+{
+ rc_t rc = 0;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+ if ( self -> error != NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcError, rcExists );
+ if ( path == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ * path = NULL;
+
+ if ( self -> localRc != 0 )
+ return self -> localRc;
+
+ rc = VPathAddRef ( self -> local );
+ if ( rc == 0 )
+ * path = self -> local;
+
+ return rc;
+}
+
+static
+rc_t VPathSetGetCache ( const VPathSet * self, const VPath ** path )
+{
+ rc_t rc = 0;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+ if ( self -> error != NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcError, rcExists );
+ if ( path == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+
+ * path = NULL;
+
+ if ( self -> cacheRc != 0 )
+ return self -> cacheRc;
+
+ rc = VPathAddRef ( self -> cache );
+ if ( rc == 0 )
+ * path = self -> cache;
+
+ return rc;
+}
+
+rc_t VPathSetMake ( VPathSet ** self, const EVPath * src,
+ bool singleUrl )
+{
+ VPathSet * p = NULL;
+ rc_t rc = 0;
+ rc_t r2 = 0;
+
+ assert ( self && src );
+
+ p = ( VPathSet * ) calloc ( 1, sizeof * p );
+ if ( p == NULL )
+ return RC ( rcVFS, rcPath, rcAllocating, rcMemory, rcExhausted );
+
+ if ( src -> error != NULL ) {
+ rc = KSrvErrorAddRef ( src -> error );
+ if ( rc == 0 )
+ p -> error = src -> error;
+ }
+
+ else if ( singleUrl ) {
+ VPUri_t uri_type = vpuri_invalid;
+ rc = VPathGetScheme_t ( src -> http, & uri_type );
+ if ( rc == 0 ) {
+ const VPath ** d = NULL;
+ switch ( uri_type ) {
+ case vpuri_fasp:
+ d = & p -> fasp;
+ break;
+ case vpuri_file:
+ d = & p -> file;
+ break;
+ case vpuri_http:
+ d = & p -> http;
+ break;
+ case vpuri_https:
+ d = & p -> https;
+ break;
+ default:
+ assert ( 0 );
+ return RC (
+ rcVFS, rcPath, rcConstructing, rcParam, rcIncorrect );
+ }
+
+ r2 = VPathAddRef ( src -> http );
+ if ( r2 == 0 )
+ * d = src -> http;
+ else if ( rc == 0 )
+ rc = r2;
+ }
+ }
+ else {
+ r2 = VPathAddRef ( src -> fasp );
+ if ( r2 == 0 )
+ p -> fasp = src -> fasp;
+ else if ( rc == 0 )
+ rc = r2;
+ r2 = VPathAddRef ( src -> vcFasp );
+ if ( r2 == 0 )
+ p -> cacheFasp = src -> vcFasp;
+ else if ( rc == 0 )
+ rc = r2;
+
+ r2 = VPathAddRef ( src -> file );
+ if ( r2 == 0 )
+ p -> file = src -> file;
+ else if ( rc == 0 )
+ rc = r2;
+ r2 = VPathAddRef ( src -> vcFile );
+ if ( r2 == 0 )
+ p -> cacheFile = src -> vcFile;
+ else if ( rc == 0 )
+ rc = r2;
+
+ r2 = VPathAddRef ( src -> http );
+ if ( r2 == 0 )
+ p -> http = src -> http;
+ else if ( rc == 0 )
+ rc = r2;
+ r2 = VPathAddRef ( src -> vcHttp );
+ if ( r2 == 0 )
+ p -> cacheHttp = src -> vcHttp;
+ else if ( rc == 0 )
+ rc = r2;
+
+ r2 = VPathAddRef ( src -> https );
+ if ( r2 == 0 )
+ p -> https = src -> https;
+ else if ( rc == 0 )
+ rc = r2;
+ r2 = VPathAddRef ( src -> vcHttps );
+ if ( r2 == 0 )
+ p -> cacheHttps = src -> vcHttps;
+ else if ( rc == 0 )
+ rc = r2;
+
+ r2 = VPathAddRef ( src -> s3 );
+ if ( r2 == 0 )
+ p -> s3 = src -> s3;
+ else if ( rc == 0 )
+ rc = r2;
+ r2 = VPathAddRef ( src -> vcS3 );
+ if ( r2 == 0 )
+ p -> cacheS3 = src -> vcS3;
+ else if ( rc == 0 )
+ rc = r2;
+ }
+
+ if ( rc == 0 ) {
+ atomic32_set ( & p -> refcount, 1 );
+
+ * self = p;
+ }
+ else
+ VPathSetWhack ( p );
+
+ return rc;
+}
+
+rc_t VPathSetMakeQuery ( VPathSet ** self, const VPath * local, rc_t localRc,
+ const VPath * cache, rc_t cacheRc )
+{
+ rc_t rc = 0;
+
+ VPathSet * p = NULL;
+
+ assert ( self );
+
+ p = ( VPathSet * ) calloc ( 1, sizeof * p );
+ if ( p == NULL )
+ return RC ( rcVFS, rcPath, rcAllocating, rcMemory, rcExhausted );
+
+ if ( localRc == 0 ) {
+ rc = VPathAddRef ( local );
+ if ( rc == 0 )
+ p -> local = local;
+ }
+ else
+ p -> localRc = localRc;
+
+ if ( cacheRc == 0 ) {
+ rc = VPathAddRef ( cache );
+ if ( rc == 0 )
+ p -> cache = cache;
+ }
+ else
+ p -> cacheRc = cacheRc;
+
+ if ( rc == 0 ) {
+ atomic32_set ( & p -> refcount, 1 );
+
+ * self = p;
+ }
+ else
+ VPathSetWhack ( p );
+
+ return rc;
+}
+
+/* KSrvResponse */
+rc_t KSrvResponseMake ( KSrvResponse ** self ) {
+ KSrvResponse * p = ( KSrvResponse * ) calloc ( 1, sizeof * p );
+ if ( p == NULL )
+ return RC ( rcVFS, rcPath, rcAllocating, rcMemory, rcExhausted );
+
+ atomic32_set ( & p -> refcount, 1 );
+
+ assert ( self );
+
+ * self = p;
+
+ return 0;
+}
+
+rc_t KSrvResponseAddRef ( const KSrvResponse * self ) {
+ if ( self != NULL )
+ atomic32_inc ( & ( ( KSrvResponse * ) self ) -> refcount );
+
+ return 0;
+}
+
+rc_t KSrvResponseRelease ( const KSrvResponse * cself ) {
+ KSrvResponse * self = ( KSrvResponse * ) cself;
+
+ if ( self != NULL && atomic32_dec_and_test ( & self -> refcount ) ) {
+ VectorWhack ( & self -> list, whackVPathSet, NULL );
+ memset ( self, 0, sizeof * self );
+ free ( self );
+ }
+
+ return 0;
+}
+
+rc_t KSrvResponseAppend ( KSrvResponse * self, const VPathSet * set ) {
+ rc_t rc = 0;
+
+ assert ( self );
+
+ rc = VPathSetAddRef ( set );
+
+ if ( rc == 0 )
+ rc = VectorAppend ( & self -> list, NULL, set );
+
+ return rc;
+}
+
+rc_t KSrvResponseAddLocalAndCache ( KSrvResponse * self, uint32_t idx,
+ const VPathSet * localAndCache )
+{
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ if ( localAndCache == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+ else {
+ VPathSet * s = ( VPathSet * ) VectorGet ( & self -> list, idx );
+ if ( s == NULL )
+ return RC ( rcVFS, rcPath, rcAccessing, rcItem, rcNotFound );
+ else {
+ rc_t rc = 0;
+ RELEASE ( VPath, s -> local );
+ if ( rc == 0 ) {
+ if ( localAndCache -> localRc == 0 ) {
+ rc = VPathAddRef ( localAndCache -> local );
+ if ( rc == 0 )
+ s -> local = localAndCache -> local;
+ }
+ else
+ s -> localRc = localAndCache -> localRc;
+ }
+ RELEASE ( VPath, s -> cache );
+ if ( rc == 0 ) {
+ if ( localAndCache -> cacheRc == 0 ) {
+ rc = VPathAddRef ( localAndCache -> cache );
+ if ( rc == 0 )
+ s -> cache = localAndCache -> cache;
+ }
+ else
+ s -> cacheRc = localAndCache -> cacheRc;
+ }
+ return rc;
+ }
+ }
+}
+
+uint32_t KSrvResponseLength ( const KSrvResponse * self ) {
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ return VectorLength ( & self -> list );
+}
+
+rc_t KSrvResponseGet
+ ( const KSrvResponse * self, uint32_t idx, const VPathSet ** set )
+{
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ if ( set == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcParam, rcNull );
+ else {
+ const VPathSet * s = ( VPathSet * ) VectorGet ( & self -> list, idx );
+ if ( s == NULL )
+ return RC ( rcVFS, rcPath, rcAccessing, rcItem, rcNotFound );
+ else {
+ rc_t rc = VPathSetAddRef ( s );
+ if ( rc == 0 )
+ * set = s;
+ return rc;
+ }
+ }
+}
+
+rc_t KSrvResponseGetPath ( const KSrvResponse * self, uint32_t idx,
+ VRemoteProtocols p, const VPath ** path, const VPath ** vdbcache,
+ const KSrvError ** error )
+{
+ const VPathSet * s = NULL;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ s = ( VPathSet * ) VectorGet ( & self -> list, idx );
+
+ if ( p == eProtocolDefault )
+ p = DEFAULT_PROTOCOLS;
+
+ if ( s == NULL )
+ return RC ( rcVFS, rcPath, rcAccessing, rcItem, rcNotFound );
+ else {
+ if ( path != NULL )
+ * path = NULL;
+ if ( vdbcache != NULL )
+ * vdbcache = NULL;
+ if ( error != NULL )
+ * error = NULL;
+ if ( s -> error == NULL )
+ return VPathSetGet ( s, p, path, vdbcache );
+ else {
+ if ( error != NULL ) {
+ rc_t rc = KSrvErrorAddRef ( s -> error );
+ if ( rc == 0 )
+ * error = s -> error;
+ return rc;
+ }
+ return RC ( rcVFS, rcQuery, rcExecuting, rcError, rcExists );
+ }
+ }
+}
+
+rc_t KSrvResponseGetLocal ( const KSrvResponse * self, uint32_t idx,
+ const VPath ** path )
+{
+ const VPathSet * s = NULL;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ s = ( VPathSet * ) VectorGet ( & self -> list, idx );
+
+ if ( s == NULL )
+ return RC ( rcVFS, rcPath, rcAccessing, rcItem, rcNotFound );
+ else {
+ if ( path != NULL )
+ * path = NULL;
+ if ( s -> error == NULL )
+ return VPathSetGetLocal ( s, path );
+ else {
+ rc_t erc = 0;
+ rc_t rc = KSrvErrorRc ( s -> error, & erc );
+ return rc == 0 ? erc : rc;
+ }
+ }
+}
+
+rc_t KSrvResponseGetCache ( const KSrvResponse * self, uint32_t idx,
+ const VPath ** path )
+{
+ const VPathSet * s = NULL;
+
+ if ( self == NULL )
+ return RC ( rcVFS, rcQuery, rcExecuting, rcSelf, rcNull );
+
+ s = ( VPathSet * ) VectorGet ( & self -> list, idx );
+
+ if ( s == NULL )
+ return RC ( rcVFS, rcPath, rcAccessing, rcItem, rcNotFound );
+ else {
+ if ( path != NULL )
+ * path = NULL;
+ if ( s -> error == NULL )
+ return VPathSetGetCache ( s, path );
+ else {
+ rc_t erc = 0;
+ rc_t rc = KSrvErrorRc ( s -> error, & erc );
+ return rc == 0 ? erc : rc;
+ }
+ }
+}
+
+/******************************************************************************/
diff --git a/py_vdb/L1-manager.py b/py_vdb/L1-manager.py
new file mode 100755
index 0000000..81064cb
--- /dev/null
+++ b/py_vdb/L1-manager.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+lib_262_rd = "./2.6.2/libncbi-vdb.so.2.6.2"
+lib_262_wr = "./2.6.2/libncbi-wvdb.so.2.6.2"
+
+lib_263_rd = "./2.6.3/libncbi-vdb.so.2.6.3"
+lib_263_wr = "./2.6.3/libncbi-wvdb.so.2.6.3"
+
+lib_270_rd = "./2.7.0/libncbi-vdb.so.2.7.0"
+lib_270_wr = "./2.7.0/libncbi-wvdb.so.2.7.0"
+
+lib_280_rd = "./2.8.0/libncbi-vdb.so.2.8.0"
+lib_280_wr = "./2.8.0/libncbi-wvdb.so.2.8.0"
+
+acc = "SRR000001"
+
+if __name__ == '__main__' :
+
+ rd_libs = [ lib_262_rd, lib_263_rd, lib_270_rd, lib_280_rd ]
+ wr_libs = [ lib_262_wr, lib_263_wr, lib_270_wr, lib_280_wr ]
+
+ for lib in rd_libs :
+ try :
+ mgr = manager( OpenMode.Read, lib )
+ print( "%s\tmgr.Version() = %s\tmgr.writable() = %r"%( lib, mgr.Version(), mgr.writable() ) )
+ except vdb_error as e :
+ print( e )
+
+ for lib in wr_libs :
+ try :
+ mgr = manager( OpenMode.Write, lib )
+ print( "%s\tmgr.Version() = %s\tmgr.writable() = %r"%( lib, mgr.Version(), mgr.writable() ) )
+ except vdb_error as e :
+ print( e )
+
+ try :
+ # if the lib is omitted, the manager is looking for a lib itself
+ # based on platform/OpenMode in the current directory first, then in $(HOME)/.ncbi/lib64
+ # if no lib can be found or the lib cannot be loaded, then the constructor throws an exception
+ mgr = manager()
+
+ mv = mgr.Version()
+ pt = mgr.PathType( acc )
+ ov = mgr.GetObjVersion( acc )
+ ot = mgr.GetObjModDate( acc )
+ print( "mgr.vers\t%s\t%s\t%s\t Version %s\tObjModDate %s"%( mv, acc, pt, ov, ot ) )
+
+ repo_mgr = mgr.MakeKConfig().MakeRepositoryMgr()
+ print( "repo_mgr.HasRemoteAccess ... %r"%( repo_mgr.HasRemoteAccess() ) )
+ for repo_list in repo_mgr.AllRepos() :
+ for repo in repo_list :
+ print( repo )
+
+ except vdb_error as e :
+ print( e )
diff --git a/py_vdb/L10-fastq.py b/py_vdb/L10-fastq.py
new file mode 100755
index 0000000..b17c086
--- /dev/null
+++ b/py_vdb/L10-fastq.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+import sys, argparse, vdb
+
+PY3 = sys.version_info[ 0 ] == 3
+
+if PY3 :
+ def xrange( *args, **kwargs ) :
+ return iter( range( *args, **kwargs ) )
+
+def fastq_from_tbl( args, tbl ) :
+ acc = args.accession[ 0 ]
+
+ col_names = [ "READ", "(INSDC:quality:text:phred_33)QUALITY", "NAME" ]
+ if args.split :
+ col_names.append( "READ_START" )
+ col_names.append( "READ_LEN" )
+ cols = tbl.CreateCursor().OpenColumns( col_names )
+
+ c_read = cols[ col_names[ 0 ] ]
+ c_qual = cols[ col_names[ 1 ] ]
+ c_name = cols[ col_names[ 2 ] ]
+ if args.split :
+ c_read_start = cols[ col_names[ 3 ] ]
+ c_read_len = cols[ col_names[ 4 ] ]
+
+ first, count = c_read.row_range()
+
+ if args.first != None :
+ first = args.first[ 0 ]
+ if args.count != None :
+ count = args.count[ 0 ]
+
+ if args.split :
+ fastq = '@{0}.{1}.{2} length={3}\n{4}\n+{0}.{1}.{2} length={3}\n{5}'
+ for row in xrange( first, first + count ) :
+ name = c_name.Read( row )
+ read = c_read.Read( row )
+ qual = c_qual.Read( row )
+ rd_start = c_read_start.Read( row )
+ rd_len = c_read_len.Read( row )
+ for x in xrange( 0, len( rd_start ) ) :
+ rlen = rd_len[ x ]
+ if rlen > 0 :
+ start = rd_start[ x ]
+ end = start + rlen
+ print( fastq.format( acc, name, x+1, rlen, read[ start:end ], qual[ start:end ] ) )
+ else :
+ fastq = '@{0}.{1} length={2}\n{3}\n+{0}.{1} length={2}\n{4}'
+ for row in xrange( first, first + count ) :
+ read = c_read.Read( row )
+ print( fastq.format( acc, c_name.Read( row ), len( read ), read, c_qual.Read( row ) ) )
+
+
+if __name__ == '__main__' :
+ parser = argparse.ArgumentParser()
+ parser.add_argument( 'accession', nargs='*' )
+ parser.add_argument( '-X', '--first', metavar='row-id', help='first row-id', nargs=1, type=int, dest='first' )
+ parser.add_argument( '-N', '--count', metavar='rows', help='how many reads', nargs=1, type=int, dest='count' )
+ parser.add_argument( '--split', help='split spot', action='store_true' )
+ args = parser.parse_args()
+
+ try :
+ #open a manager in read-mode ( dflt-mode )
+ mgr = vdb.manager()
+
+ for acc in args.accession :
+ #detect path-type ( database or table or anything-else )
+ pt = mgr.PathType( acc )
+ if pt == vdb.PathType.Database :
+ #object is a database
+ fastq_from_tbl( args, mgr.OpenDB( acc ).OpenTable( "SEQUENCE" ) )
+ elif pt == vdb.PathType.Table :
+ #object is a table
+ fastq_from_tbl( args, mgr.OpenTable( acc ) )
+ else :
+ print( "%s is not an SRA-object"%( acc ) )
+ except vdb.vdb_error as e :
+ print( e )
+ except KeyboardInterrupt :
+ print( "^C" )
diff --git a/py_vdb/L2-table_read.py b/py_vdb/L2-table_read.py
new file mode 100755
index 0000000..1a8ad5f
--- /dev/null
+++ b/py_vdb/L2-table_read.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+def get_one_value_v1( mgr, acc, column_name, row_id ) :
+ try :
+ tbl = mgr.OpenTable( acc )
+ cur = tbl.CreateCursor()
+ col = cur.OpenColumns( column_name )
+ return col.Read( row_id )
+ except vdb_error as e :
+ print( e )
+
+def get_one_value_v2( mgr, acc, column_name, row_id ) :
+ try :
+ return mgr.OpenTable( acc ).CreateCursor().OpenColumns( column_name ).Read( row_id )
+ except vdb_error as e :
+ print( e )
+
+def get_readable_columns( mgr, acc ) :
+ try :
+ tbl = mgr.OpenTable( acc )
+ return tbl.ListCol()
+ except vdb_error as e :
+ print( e )
+
+def print_column_infos( mgr, acc, column_names ) :
+ try :
+ tbl = mgr.OpenTable( acc )
+ cur = tbl.CreateCursor()
+ cols = cur.OpenColumns( column_names )
+ try :
+ v = cols.itervalues()
+ except AttributeError :
+ v = cols.values()
+ for c in v :
+ print( "%s\t %s"%( c.name, c.row_range() ) )
+ print( "domain: %s, bits: %d, dim : %d"%( c.domain(), c.bits(), c.dim() ) )
+ except vdb_error as e :
+ print( e )
+
+def inspect_column_values( mgr, acc, column_names, row_id ) :
+ try :
+ tbl = mgr.OpenTable( acc )
+ cur = tbl.CreateCursor()
+ cols = cur.OpenColumns( column_names )
+
+ #retrieve a cell-value as an array of Uint32-values ( in this case a single value )
+ spot_id = cols[ "SPOT_ID" ].Read( row_id )
+ print( "SPOT_ID[1] = %s"%( spot_id ) )
+
+ #retrieve a cell-value as a string
+ read = cols[ "READ" ].Read( row_id )
+ print( "READ[1] = %s"%( read ) )
+
+ #retrieve a cell-value as an array of Uint8-values
+ q = cols[ "QUALITY" ].Read( row_id )
+ print( "QUALITY[1] = %s"%( q ) )
+ print( "avg of quality values = %f"%( sum( q ) / float( len( q ) ) ) )
+
+ except vdb_error as e :
+ print( e )
+
+if __name__ == '__main__' :
+
+ #open a manager in read-mode
+ mgr = manager()
+
+ #a long and a short way of reading a value from a table
+ print( get_one_value_v1( mgr, "SRR000002", "READ", 2000 ) )
+ print( get_one_value_v2( mgr, "SRR000002", "READ", 2000 ) )
+
+ #print readable columns
+ print( get_readable_columns( mgr, "SRR000001" ) )
+
+ #print details about some columns
+ print_column_infos( mgr, "SRR000001", [ "SPOT_ID", "READ", "QUALITY" ] )
+
+ #open multiple columns at the same time
+ inspect_column_values( mgr, "SRR000001", [ "SPOT_ID", "READ", "QUALITY" ], 1 )
diff --git a/py_vdb/L3-table_write.py b/py_vdb/L3-table_write.py
new file mode 100755
index 0000000..91d918b
--- /dev/null
+++ b/py_vdb/L3-table_write.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+lib_wr = "./2.8.0/libncbi-wvdb.so.2.8.0"
+
+schematxt = '''
+version 1;
+table A_TABLE #1.0
+{
+ column U8 C1;
+ column U32 C2;
+ column ascii C3;
+ column I16 C4;
+ column F32 C5;
+ column bool C6;
+};
+
+'''
+
+def fill_table_random( cur, cols, row_count, value_count ) :
+ for idx in xrange( 0, row_count ) :
+ cur.OpenRow()
+ cols[ "C1" ].write_rand( value_count, 100 )
+ cols[ "C2" ].write_rand( value_count, 100000 )
+ cols[ "C3" ].write_rand( value_count )
+ cols[ "C4" ].write_rand( value_count, 100000 )
+ cols[ "C5" ].write_rand( value_count, 100000 )
+ cols[ "C6" ].write_rand( value_count )
+ cur.CommitRow()
+ cur.CloseRow()
+
+def fill_table_with_values( cur, cols ) :
+ cur.OpenRow()
+ cols[ "C1" ].write( [ 1, 2, 3, 4 ] )
+ cols[ "C2" ].write( [ 1000, 1001, 1002 ] )
+ cols[ "C3" ].write( "hallo" )
+ cols[ "C4" ].write( [ 101010, 2020, 3030 ] )
+ cols[ "C5" ].write( [ 10.101, 20.202, 30.303 ] )
+ cols[ "C6" ].write( [ True, True, False, False, True ] )
+ cur.CommitRow()
+ cur.CloseRow()
+
+def fill_table_with_single_values( cur, cols ) :
+ cur.OpenRow()
+ cols[ "C1" ].write( 1 )
+ cols[ "C2" ].write( 1000 )
+ cols[ "C3" ].write( "hallo" )
+ cols[ "C4" ].write( 5544 )
+ cols[ "C5" ].write( 55.5 )
+ cols[ "C6" ].write( True )
+ cur.CommitRow()
+ cur.CloseRow()
+
+def fill_table_with_default_values( cur, cols, row_count ) :
+ cols[ "C1" ].set_default( [ 100, 101 ] )
+ for idx in xrange( 0, row_count ) :
+ cur.OpenRow()
+ cols[ "C2" ].write( [ 10 + idx, 11 + idx, 12 + idx ] )
+ cols[ "C3" ].write( "line #%d"%idx )
+ cols[ "C4" ].write( [ 101 + idx, 102 + idx, 103 + idx ] )
+ cols[ "C5" ].write( [ 10.1 + idx, 20.2 + idx, 30.3 + idx ] )
+ cols[ "C6" ].write( [ True ] )
+ cur.CommitRow()
+ cur.CloseRow()
+
+def make_table( mgr, schema_txt, table_name ) :
+ try :
+ schema = mgr.MakeSchema( schema_txt )
+ tbl = mgr.CreateTable( schema, "A_TABLE", table_name )
+ cur = tbl.CreateCursor( OpenMode.Write )
+ cols = cur.OpenColumns( [ "C1", "C2", "C3", "C4", "C5", "C6" ] )
+ fill_table_random( cur, cols, 2, 5 )
+ fill_table_with_values( cur, cols )
+ fill_table_with_single_values( cur, cols )
+ fill_table_with_default_values( cur, cols, 2 )
+ cur.Commit()
+ except vdb_error as e :
+ print( e )
+
+
+if __name__ == '__main__' :
+
+ table_name = "L3"
+
+ try :
+ #open a manager
+ mgr = manager( OpenMode.Write, lib_wr )
+
+ make_table( mgr, schematxt, table_name )
+ mgr.OpenTable( table_name ).print_rows()
+
+ except vdb_error as e :
+ print( e )
diff --git a/py_vdb/L4-database_read.py b/py_vdb/L4-database_read.py
new file mode 100755
index 0000000..0c22ab5
--- /dev/null
+++ b/py_vdb/L4-database_read.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+# ------------------------------------------------------------------------
+
+#exam the database what sub-databases, sub-tables
+def exam_db( db ) :
+ try :
+ print( "database \t: %s"% ( db.Name() ) )
+ except vdb_error as e :
+ pass
+ try :
+ print( "sub-db-s \t: %s"%( db.ListDB() ) )
+ except vdb_error as e :
+ pass
+
+# ------------------------------------------------------------------------
+
+def exam_tables( db ) :
+ try :
+ sub_tabs = db.ListTbl()
+ print( "sub-tables's \t: %s"%sub_tabs )
+ for t in sub_tabs :
+ tab = db.OpenTable( t )
+ print( "\n%s.columns = %s"%( tab.Name(), tab.ListCol() ) )
+ except vdb_error as e :
+ pass
+
+# ------------------------------------------------------------------------
+
+def exam_index( db ) :
+ try :
+ for tablename in db.ListTbl() :
+ t = db.OpenTable( tablename )
+ index_list = t.ListIdx()
+ print( "\nINDEX %s.%s : %s"%( db.Name(), tablename, index_list ) )
+ if len( index_list ) > 0 :
+ for index_name in index_list :
+ index = t.OpenIndexRead( index_name )
+ print( "Version( '%s' ) = %s"%( index_name, index.Version() ) )
+ print( "Type( '%s' ) = %s"%( index_name, index.Type() ) )
+ print( "Locked( '%s' ) = %s"%( index_name, index.Locked() ) )
+ except vdb_error as e :
+ pass
+
+# ------------------------------------------------------------------------
+
+if __name__ == '__main__' :
+ try :
+ #open database in read-mode ( default )
+ with manager().OpenDB( "SRR834507" ) as db :
+ exam_db( db )
+ exam_tables( db )
+ exam_index( db )
+
+ except vdb_error as e :
+ print( e )
diff --git a/py_vdb/L5-database_write.py b/py_vdb/L5-database_write.py
new file mode 100755
index 0000000..5c2db93
--- /dev/null
+++ b/py_vdb/L5-database_write.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+lib_wr = "./2.8.0/libncbi-wvdb.so.2.8.0"
+
+schema_txt = '''
+version 1;
+
+table SUB_TAB #1.0
+{
+ column U32 C1;
+ column ascii C2;
+};
+
+database SUB_DB #1.0
+{
+ table SUB_TAB #1 TAB20;
+};
+
+database MAIN_DB #1.0
+{
+ table SUB_TAB #1 TAB10;
+ database SUB_DB #1 SUBDB;
+};
+'''
+
+def fill_table( tbl, row_count, value_count ) :
+ try :
+ cur = tbl.CreateCursor( OpenMode.Write )
+ cols = cur.OpenColumns( [ "C1", "C2" ] )
+ for idx in xrange( 0, row_count ) :
+ cur.OpenRow()
+ cols[ "C1" ].write_rand( value_count, 100 )
+ cols[ "C2" ].write_rand( value_count, 80 )
+ cur.CommitRow()
+ cur.CloseRow()
+ cur.Commit()
+ except vdb_error as e :
+ print( e )
+
+def create_database( mgr, spec, path ) :
+ #make a schema from the text above
+ schema = mgr.MakeSchema( schema_txt )
+
+ #create a database with this schema
+ spec_in_schema = "MAIN_DB"
+ path_to_create = "L5"
+ db = mgr.CreateDB( schema, spec, path )
+
+ t1 = db.CreateTable( "TAB10" )
+ fill_table( t1, 2, 3 )
+
+ subdb = db.CreateDB( "SUBDB" )
+ t2 = subdb.CreateTable( "TAB20" )
+ fill_table( t2, 2, 3 )
+
+# ------------------------------------------------------------------------
+
+def print_tables( db, prefix ) :
+ try :
+ tables = db.ListTbl()
+ for tabname in tables :
+ try :
+ t = db.OpenTable( tabname )
+ print( "%s|---TABLE: '%s'"%( prefix, t.Name() ) )
+ t.print_rows( None, None, "%s "%( prefix ) )
+ except vdb_error as e :
+ print( e )
+ except vdb_error as e :
+ pass
+
+def print_databases( db, prefix ) :
+ print_tables( db, prefix )
+ try :
+ databases = db.ListDB()
+ for dbname in databases :
+ try :
+ d = db.OpenDB( dbname )
+ print( "%s|---DATABASE: '%s'"%( prefix, d.Name() ) )
+ print_databases( d, "%s "%( prefix ) )
+ except vdb_error as e :
+ print( e )
+ except vdb_error as e :
+ pass
+
+def print_db( db ) :
+ print( "DATABASE: '%s':"%( db.Name() ) )
+ print_databases( db, "" )
+
+# ------------------------------------------------------------------------
+
+if __name__ == '__main__' :
+
+ try :
+ #open a manager in read-mode
+ mgr = manager( OpenMode.Write, lib_wr )
+
+ #make a schema from the text above
+ schema = mgr.MakeSchema( schema_txt )
+
+ #create a database with this schema
+ spec_in_schema = "MAIN_DB"
+ path_to_create = "L5"
+ create_database( mgr, spec_in_schema, path_to_create )
+
+ #print the content of the created database
+ print_db( mgr.OpenDB( path_to_create ) )
+
+ except vdb_error as e :
+ print( e )
diff --git a/py_vdb/L6-meta_read.py b/py_vdb/L6-meta_read.py
new file mode 100755
index 0000000..6f09e11
--- /dev/null
+++ b/py_vdb/L6-meta_read.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+lib_rd = './2.8.0/libncbi-vdb.so.2.8.0'
+
+# ------------------------------------------------------------------------
+
+# ------------------------------------------------------------------------
+
+if __name__ == '__main__' :
+
+ try :
+ #open a manager in read-mode
+ mgr = manager( OpenMode.Read, lib_rd )
+
+ #open accession ( a table-accession )
+ tbl = mgr.OpenTable( 'SRR942391' )
+
+ #open meta-data on table
+ meta = tbl.OpenMetadata()
+
+ print( "version = %s"%meta.Version() )
+ print( "byte-order reversed = %r"%meta.ByteOrder() )
+ print( "revision = %d"%meta.Revision() )
+ print( "max-revision = %d"%meta.MaxRevision() )
+
+ #open the root
+ root_node = meta.OpenNode( '/' )
+ child_list = root_node.ListChildren()
+ print( "sub-nodes: %s"%child_list )
+ for node_name in child_list :
+ node = root_node.OpenNode( node_name )
+ print ( "%s.byte-order reversed = %s"%( node_name, node.ByteOrder() ) )
+ print ( "%s.size = %d"%( node_name, node.size() ) )
+ print ( "%s.data = %s"%( node_name, node.as_string( 41 ) ) )
+ print ( "%s.data (uint8) = %s"%( node_name, node.as_uint8( 41 ) ) )
+ print ( "%s.data (int8) = %s"%( node_name, node.as_int8( 41 ) ) )
+ print ( "%s.data (uint16) = %s"%( node_name, node.as_uint16( 41 ) ) )
+ print ( "%s.data (int16) = %s"%( node_name, node.as_int16( 41 ) ) )
+ print ( "%s.data (uint32) = %s"%( node_name, node.as_uint32( 41 ) ) )
+ print ( "%s.data (int32) = %s"%( node_name, node.as_int32( 41 ) ) )
+ print ( "%s.data (uint64) = %s"%( node_name, node.as_uint64( 41 ) ) )
+ print ( "%s.data (int64) = %s"%( node_name, node.as_int64( 41 ) ) )
+
+ except vdb_error as e :
+ print( e )
diff --git a/py_vdb/L7-table-rnd-write.py b/py_vdb/L7-table-rnd-write.py
new file mode 100755
index 0000000..c432968
--- /dev/null
+++ b/py_vdb/L7-table-rnd-write.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+lib_wr = "./2.8.0/libncbi-wvdb.so.2.8.0"
+
+schematxt = '''
+version 1;
+
+include 'vdb/vdb.vschema';
+
+table A_TABLE #1.0
+{
+ column < U8 > izip_encoding C1;
+};
+
+'''
+
+def fill_table_random( cur, cols, row_count, value_count ) :
+ for idx in xrange( 0, row_count ) :
+ cur.OpenRow()
+ cols[ "C1" ].write_rand( value_count, 255 )
+ cur.CommitRow()
+ cur.CloseRow()
+
+def make_table( mgr, schema_txt, table_name ) :
+ try :
+ schema = mgr.MakeSchema( schema_txt )
+ tbl = mgr.CreateTable( schema, "A_TABLE", table_name )
+ cur = tbl.CreateCursor( OpenMode.Write )
+ cols = cur.OpenColumns( [ "C1" ] )
+ fill_table_random( cur, cols, 320, 1024 )
+ cur.Commit()
+ except vdb_error as e :
+ print( e )
+
+
+if __name__ == '__main__' :
+
+ table_name = "L7"
+
+ try :
+ mgr = manager( OpenMode.Write, lib_wr )
+
+ make_table( mgr, schematxt, table_name )
+ #mgr.OpenTable( table_name ).print_rows()
+
+ except vdb_error as e :
+ print( e )
diff --git a/py_vdb/L8-import-csv.py b/py_vdb/L8-import-csv.py
new file mode 100755
index 0000000..0697460
--- /dev/null
+++ b/py_vdb/L8-import-csv.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+import vdb, csv, os
+
+schematxt = '''
+version 1;
+table CSVTAB #1.0
+{
+ column U16 C1;
+ column ascii C2;
+};
+'''
+
+def write_csv( filename, rowcount ) :
+ with open( filename, 'w' ) as f :
+ for i in range( 1, rowcount ) :
+ f.write( "%d, line # %d\n"%( i, i ) )
+ f.close()
+
+def csv_to_vdb( mgr, csv_file, spec, vdb_table ) :
+ with open( csv_file, 'r' ) as f :
+ spec = 'CSVTAB'
+ tbl_wr = mgr.CreateTable( mgr.MakeSchema( schematxt ), spec, vdb_table )
+ cur = tbl_wr.CreateCursor( vdb.OpenMode.Write )
+ cols = cur.OpenColumns( [ "C1", "C2" ] )
+ for row in csv.reader( f ) :
+ cur.OpenRow()
+ cols[ "C1" ].write( [ int( row[ 0 ] ) ] )
+ cols[ "C2" ].write( row[ 1 ].strip() );
+ cur.CommitRow()
+ cur.CloseRow()
+ cur.Commit()
+
+if __name__ == '__main__':
+ csv_file = 'data.txt'
+ write_csv( csv_file, 10 )
+ try :
+ mgr = vdb.manager( vdb.OpenMode.Write, "./2.8.0/libncbi-wvdb.so.2.8.0" )
+ spec = 'CSVTAB'
+ csv_to_vdb( mgr, csv_file, spec, 'L8' )
+ mgr.OpenTable( spec ).print_rows()
+ except vdb.vdb_error as e :
+ print( e )
+ os.remove( csv_file )
diff --git a/py_vdb/L9-index_usage.py b/py_vdb/L9-index_usage.py
new file mode 100755
index 0000000..d8822f0
--- /dev/null
+++ b/py_vdb/L9-index_usage.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+lib_wr = "./2.8.0/libncbi-wvdb.so.2.8.0"
+
+schematxt = '''
+version 1;
+table A_TABLE #1.0
+{
+ column U32 C1;
+ column ascii C2;
+};
+
+'''
+
+def fill_table_random( cur, cols, kidx, row_count, value_count ) :
+ for idx in xrange( 0, row_count ) :
+ cur.OpenRow()
+ row_id = cur.RowId()
+ cols[ "C1" ].write_rand( value_count, 0xffffffff )
+ rand_str = random_string( value_count )
+ cols[ "C2" ].write( rand_str )
+ cur.CommitRow()
+ cur.CloseRow()
+ #print( "Row = %d"%row_id )
+ if ( idx % 1000 ) == 0 :
+ kidx.InsertText( True, rand_str, row_id )
+
+def make_table( mgr, schema_txt, table_spec, table_name ) :
+ try :
+ schema = mgr.MakeSchema( schema_txt )
+ tbl = mgr.CreateTable( schema, table_spec, table_name )
+ kidx = tbl.CreateIndex( "C2_IDX", IndexType.Text )
+ cur = tbl.CreateCursor( OpenMode.Write )
+ cols = cur.OpenColumns( [ "C1", "C2" ] )
+ fill_table_random( cur, cols, kidx, 200000, 25 )
+ kidx.Commit()
+ cur.Commit()
+ except vdb_error as e :
+ print( e )
+
+
+if __name__ == '__main__' :
+
+ table_name = 'L9'
+ table_spec = 'A_TABLE'
+
+ try :
+ #open a manager
+ mgr = manager( OpenMode.Write, lib_wr )
+
+ make_table( mgr, schematxt, table_spec, table_name )
+ #mgr.OpenTable( table_name ).print_rows()
+
+ except vdb_error as e :
+ print( e )
diff --git a/py_vdb/custom_fastq.py b/py_vdb/custom_fastq.py
new file mode 100755
index 0000000..c991f45
--- /dev/null
+++ b/py_vdb/custom_fastq.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+
+if __name__ == '__main__':
+ lib_262_r = "./2.8.0/libncbi-vdb.so.2.8.0"
+ tablename = "SRR000001"
+
+ try :
+ mgr = manager( OpenMode.Read, lib_262_r )
+ tab = mgr.open_tab( tablename )
+ cur = tab.make_cursor( OpenMode.Read )
+ cols = cur.open( [ "NAME", "READ", "(INSDC:quality:text:phred_33)QUALITY" ] )
+ c_name = cols[ "NAME" ]
+ c_read = cols[ "READ" ]
+ c_qual = cols[ "(INSDC:quality:text:phred_33)QUALITY" ]
+ for row in xrange( 1, 11 ) :
+ name = c_name.read( row )
+ read = c_read.read( row )
+ print "@%s.%d %s lenght=%d"%( tablename, row, name, len( read ) )
+ print read
+ print "+%s.%d %s lenght=%d"%( tablename, row, name, len( read ) )
+ print c_qual.read( row )
+
+ except vdb_error, e :
+ print e
diff --git a/py_vdb/ref_var.py b/py_vdb/ref_var.py
new file mode 100755
index 0000000..058c91b
--- /dev/null
+++ b/py_vdb/ref_var.py
@@ -0,0 +1,375 @@
+#!/usr/bin/env python
+
+from vdb import *
+import multiprocessing, sys
+
+'''
+def f1_a( mgr, ref, del_pos, del_len, ins ) :
+ del_pos = 2
+ del_len = 1
+ ins = "CCAA"
+
+ print "ref = '%s' del: %d:%d ins: '%s'" % ( ref, del_pos, del_len, ins )
+
+ ref_var = mgr.make_ref_var( ref, del_pos, del_len, ins )
+
+ #q = ref_var.search()
+ #qr = ref_var.search_len()
+ #print "query = '%s' at: %d:%d, on ref: %d" %( q[0], q[2], q[1], qr )
+
+ ( a_bases, a_len, a_pos ) = ref_var.allele()
+ ar = ref_var.allele_len()
+ print "allele = '%s' at: %d:%d, on ref: %d" %( a_bases, a_pos, a_len, ar )
+
+#------------------------------------------------------------------------------------------------------------
+def f1( mgr ):
+ ref = "ACCGGTTAACC"
+
+ del_pos = 2
+ del_len = 1
+ ins = "CCAA"
+
+ f1_a( mgr, ref, del_pos, del_len, ins )
+ f1_a( mgr, ref, del_pos, del_len, ins )
+
+#------------------------------------------------------------------------------------------------------------
+def f2( mgr, acc ) :
+ refs = mgr.make_reflist( acc )
+ count = refs.count()
+ print "we have %d references:" % count
+ for idx in xrange( count ) :
+ print "\t No. %d" % idx
+ obj = refs.get( idx )
+ #print "\t\tidx:\t%d" % obj.get_idx()
+ rr = obj.get_row_range()
+ print "\t\trows:\t%d..%d" % ( rr[0], rr[1] )
+ #print "\t\tbin:\t%d" % obj.get_bin()
+ print "\t\tSeqId:\t%s" % obj.get_seq_id()
+ print "\t\tname:\t%s" % obj.get_name()
+ print "\t\tlength:\t%d" % obj.get_length()
+ print "\t\tcirc:\t%s" % obj.is_circular()
+ print "\t\text:\t%s" % obj.is_external()
+ #print "\t\tdata:\t%s" % obj.read( 0, 50 )
+ #print "\t\tid-cnt:\t%d" % obj.id_count( rr[ 0 ] )
+
+
+#------------------------------------------------------------------------------------------------------------
+def num( s ):
+ try:
+ return int( s )
+ except ValueError:
+ return 0
+
+
+#------------------------------------------------------------------------------------------------------------
+def split_cigar( cigar ) :
+ res = list()
+ op_len = ""
+ for i in xrange( 0, len( cigar ) ) :
+ op = cigar[ i ]
+ if op >= '0' and op <= '9' :
+ op_len = op_len + op
+ else :
+ tup = ( num( op_len ), op )
+ op_len = ""
+ res.append( tup )
+ return res
+
+
+#------------------------------------------------------------------------------------------------------------
+def f3( mgr, acc ) :
+ cur = mgr.open_db( acc ).open_tab( "PRIMARY_ALIGNMENT" ).make_cursor()
+ cols = cur.open( [ "CIGAR_SHORT", "READ", "REF_SEQ_ID", "REF_POS", "REF_LEN" ] )
+ row = read_row( cols, 1 )
+ print row
+
+
+#------------------------------------------------------------------------------------------------------------
+def f4( mgr, acc ) :
+ cur = mgr.open_db( acc ).open_tab( "PRIMARY_ALIGNMENT" ).make_cursor()
+ cols = cur.open( [ "CIGAR_SHORT", "READ", "REF_SEQ_ID", "REF_POS", "REF_LEN" ] )
+ row_range = cols[ "READ" ].range()
+ print row_range
+ for row in row_range :
+ row_data = read_row( cols, row )
+ if row % 1000 == 0 :
+ sys.stdout.write( '.' )
+ sys.stdout.flush()
+
+#------------------------------------------------------------------------------------------------------------
+def cigar_splitter( cigar, ref_pos, reference ) :
+ ops = split_cigar( cigar )
+
+
+#------------------------------------------------------------------------------------------------------------
+def handle_reference( ref_obj, total, prim_alig_id_col, prim_cols, cigars ) :
+ res = total
+ ref_rows = ref_obj.get_row_range()
+ ref_len = ref_obj.get_length()
+ name = ref_obj.get_seq_id()
+ #read the whole reference in...
+ reference = ref_obj.read( 0, ref_len )
+ print "\n", name, ref_len, len( reference )
+ # for each row in the reference-table of this reference
+ for ref_row in xrange( ref_rows[ 0 ], ref_rows[ 1 ] + 1 ) :
+ prim_ids = prim_alig_id_col.read( ref_row )
+ #for each alignment in this reference-block
+ for prim_id in prim_ids :
+ row_data = read_row( prim_cols, prim_id )
+ cigar = row_data[ "CIGAR_SHORT" ]
+ if cigar in cigars.keys() :
+ cigars[ cigar ] += 1
+ else :
+ cigars[ cigar ] = 1
+ res += 1
+ if res % 1000 == 0 :
+ sys.stdout.write( '.' )
+ sys.stdout.flush()
+ return res
+
+
+
+#------------------------------------------------------------------------------------------------------------
+def f5( mgr, acc, ref_idx = None ) :
+ db = mgr.open_db( acc )
+ cur_a = db.open_tab( "PRIMARY_ALIGNMENT" ).make_cursor()
+ cur_r = db.open_tab( "REFERENCE" ).make_cursor()
+ refs = db.make_reflist()
+ prim_cols = cur_a.open( [ "CIGAR_SHORT", "READ", "REF_SEQ_ID", "REF_POS", "REF_LEN" ] )
+ prim_alig_id_col = cur_r.open( "PRIMARY_ALIGNMENT_IDS" )
+ total = 0
+ cigars = {}
+
+ if ref_idx == None :
+ # for each reference
+ for idx in xrange( refs.count() ) :
+ total += handle_reference( refs.get( idx ), total, prim_alig_id_col, prim_cols, cigars )
+ else :
+ total += handle_reference( refs.get( ref_idx ), total, prim_alig_id_col, prim_cols, cigars )
+
+ print "\nhandled ", total, " alignments"
+ print "we have ", len( cigars ), " different cigar-strings"
+ sorted_cigars = sorted( cigars, key = cigars.get, reverse = True )
+ for w in sorted_cigars[ 0 : 10 ] :
+ print w, cigars[ w ]
+
+'''
+
+#------------------------------------------------------------------------------------------------------------
+def cigar2events( cigar ) :
+ res = list()
+ tmp = ""
+ for c in cigar :
+ if c >= '0' and c <= '9' :
+ tmp += c
+ else :
+ res.append( ( int( tmp ), c ) )
+ tmp = ""
+ return res
+
+
+#------------------------------------------------------------------------------------------------------------
+def events2dict( events ) :
+ res = {}
+ for ( len, op ) in events :
+ res[ op ] = res.get( op, 0 ) + 1
+ return res
+
+
+#------------------------------------------------------------------------------------------------------------
+def adjacent( events, op1, op2 ) :
+ res = 0
+ last = 'x'
+ for ( len, op ) in events :
+ if op == op1 and last == op2 :
+ res += 1
+ if op == op2 and last == op1 :
+ res += 1
+ last = op
+ return res
+
+
+#------------------------------------------------------------------------------------------------------------
+def cigar_events( cigar, read, refname, pos ) :
+ ref_pos = pos
+ ali_pos = 0
+ events = cigar2events( cigar )
+ for ( len, op ) in events :
+ if op == '=' : # we have a perfect match between reference and alignment '='
+ ref_pos += len # we advance on alignment AND reference
+ ali_pos += len
+
+ elif op == 'X' : #we have a mismatch between reference and alignment 'X'
+ yield ( refname, ref_pos, 0, read[ ali_pos : ali_pos + len ] )
+ ref_pos += len # we advance on alignment AND reference
+ ali_pos += len
+
+ elif op == 'D' : #we have a deletion on the reference 'D'
+ yield ( refname, ref_pos, len, '' )
+ ref_pos += len # we advance only on reference
+
+ elif op == 'I' : #we have a insertion on the reference 'I'
+ yield ( refname, ref_pos, 0, read[ ali_pos : ali_pos + len ] )
+ ali_pos += len # we advance only on alignment
+
+ elif op == 'S' : #we have a soft clip 'S'
+ ali_pos += len # we advance only on alignment
+
+ elif op == 'H' : #we have a hard clip 'H'
+ ali_pos += 0 # we do nothing
+
+ elif op == 'N' : #we have a reference 'N'
+ ref_pos += len # we advance on alignment AND reference
+
+ else :
+ ali_pos += 0 # we do nothing
+
+#------------------------------------------------------------------------------------------------------------
+# a generator of alleles...
+def alleles( db, row_range = None ) :
+ column_list = [ 'CIGAR_LONG', 'READ', 'REF_SEQ_ID', 'REF_POS', 'REF_LEN' ]
+ prim_cols = db.open_tab( "PRIMARY_ALIGNMENT" ).make_cursor().open( column_list )
+ for row in row_gen( prim_cols, row_range ) :
+ cigar = row[ 'CIGAR_LONG' ]
+ read = row[ 'READ' ]
+ refname = row[ 'REF_SEQ_ID' ]
+ refpos = row[ 'REF_POS' ][ 0 ]
+ reflen = row[ 'REF_LEN' ][ 0 ]
+ for c in cigar_events( cigar, read, refname, refpos, reflen ) :
+ yield c
+ print
+
+#------------------------------------------------------------------------------------------------------------
+def refvar_consumer( q, filename ) :
+ print( "refvar_consumer() started" )
+ d = {}
+ while True :
+ signature = q.get()
+ if signature == None :
+ break
+ d[ signature ] = d.get( signature, 0 ) + 1
+
+ f = open( filename, "w" )
+ for k, v in sorted( [ ( value, key ) for ( key, value ) in d.items() ], reverse=True ) :
+ f.write( "%d %s\n" %( k, v ) )
+ f.close()
+ print( "refvar_consumer() done" )
+
+
+#------------------------------------------------------------------------------------------------------------
+def allel_consumer( mgr, acc, q_in, q_out1, q_out2 ) :
+ print( "allel_consumer() started" )
+ try :
+ ref_list = mgr.OpenDB( acc ).ReferenceList()
+ curr_ref = None
+ ref_bases = None
+ while True :
+ t = q_in.get()
+ if t == None :
+ break
+ ( ref_name, ref_pos, del_len, bases ) = t
+ if curr_ref == None or curr_ref != ref_name :
+ curr_ref = ref_name
+ try :
+ ref_obj = ref_list.find( ref_name )
+ ref_bases = ref_obj.Read( 0, ref_obj.SeqLength() )[:]
+ except vdb_error as e :
+ print( e )
+
+ sig1 = "%s:%d:%d:%s" % ( ref_name, ref_pos, del_len, bases )
+ q_out1.put( sig1 )
+
+ if len( bases ) > 0 :
+ # we have insertion/mismatch : let's canonicalize it
+ ref_var = mgr.RefVariation( ref_bases, ref_pos, del_len, bases )
+ ( a_bases, a_len, a_pos ) = ref_var.GetAllele()
+ sig2 = "%s:%d:%d:%s" % ( ref_name, a_pos, a_len, a_bases )
+ else :
+ # we have a pure deletion
+ sig2 = sig1
+
+ q_out2.put( sig2 )
+ except vdb_error as e :
+ print( e )
+ q_out1.put( None )
+ q_out2.put( None )
+ print( "allel_consumer() done" )
+
+
+#------------------------------------------------------------------------------------------------------------
+def row_consumer_allel_producer( q_in, q_out ) :
+ print( "row_consumer() started" )
+ while True :
+ row = q_in.get()
+ if row == None :
+ break
+ try :
+ cigar = row[ 'CIGAR_LONG' ]
+ read = row[ 'READ' ]
+ refname = row[ 'REF_SEQ_ID' ]
+ refpos = row[ 'REF_POS' ][ 0 ]
+ for c in cigar_events( cigar, read, refname, refpos ) :
+ q_out.put( c )
+ except vdb_error as e :
+ print( e )
+
+ print( "row_consumer() done" )
+ q_out.put( None )
+
+
+#------------------------------------------------------------------------------------------------------------
+def row_producer( mgr, acc, row_range, q ) :
+ try :
+ print( "row_producer() started" )
+ db = mgr.OpenDB( acc )
+ cols = [ 'CIGAR_LONG', 'READ', 'REF_SEQ_ID', 'REF_POS' ]
+ prim_cols = db.OpenTable( "PRIMARY_ALIGNMENT" ).CreateCursor().OpenColumns( cols )
+ for row in row_gen( prim_cols, row_range ) :
+ q.put( row )
+ q.put( None )
+ print( "row_producer() done" )
+ except vdb_error as e :
+ print( e )
+
+
+def process_accession( mgr, acc, row_range = None ) :
+ row_q = multiprocessing.Queue()
+
+ p_row_producer = multiprocessing.Process( target = row_producer, args = ( mgr, acc, row_range, row_q ), )
+
+ allel_q = multiprocessing.Queue()
+
+ p_row_cons = multiprocessing.Process( target = row_consumer_allel_producer, args = ( row_q, allel_q ), )
+
+ refvar_q1 = multiprocessing.Queue()
+ refvar_q2 = multiprocessing.Queue()
+
+ p_allel_cons = multiprocessing.Process( target = allel_consumer, args = ( mgr, acc, allel_q, refvar_q1, refvar_q2 ), )
+
+ p_refvar1 = multiprocessing.Process( target = refvar_consumer, args = ( refvar_q1, "ref_var_1.txt" ), )
+ p_refvar2 = multiprocessing.Process( target = refvar_consumer, args = ( refvar_q2, "ref_var_2.txt" ), )
+
+ p_row_producer.start()
+ p_row_cons.start()
+ p_allel_cons.start()
+ p_refvar1.start()
+ p_refvar2.start()
+
+ p_row_producer.join()
+ p_row_cons.join()
+ p_allel_cons.join()
+ p_refvar1.join()
+ p_refvar2.join()
+
+
+#------------------------------------------------------------------------------------------------------------
+if __name__ == '__main__':
+ lib_r = "./2.8.0/libncbi-vdb.so.2.8.0"
+ ACC = "SRR1531793"
+
+ if len( sys.argv ) > 1 :
+ ACC = sys.argv[ 1 ]
+
+ mgr = manager( OpenMode.Read, lib_r )
+ process_accession( mgr, ACC )
+ #f1( mgr )
diff --git a/py_vdb/tst_config.py b/py_vdb/tst_config.py
new file mode 100755
index 0000000..8cb6adf
--- /dev/null
+++ b/py_vdb/tst_config.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+from vdb import *
+
+def print_repolist( repos ) :
+ for r in repos :
+ name = r.name()
+ print "%s.displayname = %s"%( name, r.display_name() )
+ print "%s.category = %s"%( name, RepoCat2String( r.category() ) )
+ print "%s.subcategory = %s"%( name, RepoSubCat2String( r.sub_category() ) )
+ print "%s.root = %s"%( name, r.get_root() )
+ print "%s.resolver = %s"%( name, r.resolver() )
+ print "%s.disabled = %s"%( name, r.is_disabled() )
+ print "%s.cache enabled = %s"%( name, r.cache_enabled() )
+ print "-"
+
+def report_repos( rm ) :
+ print "user repos:"
+ print_repolist( rm.user_repos() )
+ print "site repos:"
+ print_repolist( rm.site_repos() )
+ print "remote repos:"
+ print_repolist( rm.remote_repos() )
+
+if __name__ == '__main__':
+ lib_263_rd = "./2.6.3/libncbi-vdb.so.2.6.3"
+ lib_262_rd = "./2.6.2/libncbi-vdb.so.2.6.2"
+ lib_rd = "/home/raetzw/.ncbi/lib64/libncbi-vdb.so"
+
+ mgr = manager( OpenMode.Read, lib_262_rd )
+ cfg = mgr.make_config()
+
+ rm = cfg.make_repo_mgr()
+ print "has remote access = %s"%( rm.has_remote_access() )
+
+ #report_repos( rm )
+
+ print "user.disabled = %s"%( rm.cat_disabled( RepoCat.User ) )
+ print "site.disabled = %s"%( rm.cat_disabled( RepoCat.Site ) )
+ print "remote.disabled = %s"%( rm.cat_disabled( RepoCat.Remote ) )
+
+ vfs_mgr = mgr.make_vfs_mgr()
+ resolv = vfs_mgr.make_resolver( cfg )
+ #resolv. remote_enable( ResolvEnable.AlwaysEnable )
+
+ print "local = %s"%( resolv.query_local( vfs_mgr.make_path( "ncbi-acc:SRR001000" ) ) )
+ print "remote = %s"%( resolv.query_remote( vfs_mgr.make_path( "ncbi-acc:SRR001000" ) ) )
+ print "cache = %s"%( resolv.query_cache( vfs_mgr.make_path( "ncbi-acc:SRR001000" ) ) )
diff --git a/py_vdb/vdb.py b/py_vdb/vdb.py
new file mode 100644
index 0000000..f96a35f
--- /dev/null
+++ b/py_vdb/vdb.py
@@ -0,0 +1,2229 @@
+from ctypes import *
+from enum import Enum
+import datetime, string, random, sys, os, platform
+
+PY3 = sys.version_info[ 0 ] == 3
+
+if PY3 :
+ def xrange( *args, **kwargs ) :
+ return iter( range( *args, **kwargs ) )
+
+def to_bytes( s ) :
+ if PY3 and ( type( s ) == str ) :
+ return str.encode( s )
+ return s
+
+def to_char_p( s ) :
+ if PY3 and ( type( s ) == str ) :
+ return c_char_p( str.encode( s ) )
+ return c_char_p( s )
+
+
+class CreateMode( Enum ) :
+ Open = 0
+ Init = 1
+ Create = 2
+ MD5 = ( 1 << 6 )
+
+class TypeDomain( Enum ) :
+ Bool = 1
+ UInt = 2
+ Int = 3
+ Float = 4
+ Ascii = 5
+ Unicode = 6
+
+class CursorMode( Enum ) :
+ Update = 0
+ Replace = 1
+ Insert = 2
+
+class OpenMode( Enum ) :
+ Read = 0
+ Write = 1
+
+class RepoCat( Enum ) :
+ Bad = 0
+ User = 1
+ Site = 2
+ Remote = 3
+
+class RepoSubCat( Enum ) :
+ Bad = 0
+ Main = 1
+ Aux = 2
+ Protected = 3
+
+class RemoteProto( Enum ) :
+ Http = 0
+ Fasp = 1
+ FaspHttp = 2
+ HttpFasp = 3
+
+class ResolvEnable( Enum ) :
+ UseConfig = 0
+ AlwaysEnable = 1
+ AlwaysDisable = 2
+
+class PathType( Enum ) :
+ NotFound = 0
+ BadPath = 1
+ File = 2
+ Dir = 3
+ CharDev = 4
+ BlockDev = 5
+ Fifo = 6
+ ZombieFile = 7
+ FakeRoot = 8
+ Dataset = 9
+ Datatype = 10
+ Database = 11
+ Table = 12
+ Index = 13
+ Column = 14,
+ Metadata = 15
+ PrereleaseTbl = 16
+ Alias = 128
+
+class IndexType( Enum ) :
+ Text = 0
+ UInt64 = 1
+ Text_reverse = 128
+ UInt64_reverse = 129
+
+RepoCatDict = { RepoCat.Bad : "Bad", RepoCat.User : "User", RepoCat.Site : "Site", RepoCat.Remote : "Remote" }
+def RepoCat2String( cat ) :
+ x = RepoCatDict[ cat ]
+ if x == None :
+ x = "Bad"
+ return x
+
+RepoSubCatDict = { RepoSubCat.Bad : "Bad", RepoSubCat.Main : "Main", RepoSubCat.Aux : "Aux", RepoSubCat.Protected : "Protected" }
+def RepoSubCat2String( cat ) :
+ x = RepoSubCatDict[ cat ]
+ if x == None :
+ x = "Bad"
+ return x
+
+class vdb_string( Structure ) :
+ _fields_ = [ ( "addr", c_char_p ), ( "size", c_int ), ( "len", c_int ) ]
+
+ def __str__( self ) :
+ return self.addr.value
+
+class vdb_vector( Structure ) :
+ _fields_ = [ ( "v", c_void_p ), ( "start", c_int ), ( "len", c_int ), ( "mask", c_int ) ]
+
+
+#------------------------------------------------------------------------------------------------------------
+class version :
+ major = 0
+ minor = 0
+ release = 0
+
+ def __init__( self, s ) :
+ if PY3 :
+ string_types = str
+ else :
+ string_types = basestring
+ if isinstance( s, string_types ) :
+ a = s.split( '.' )
+ l = len( a )
+ if l > 0 :
+ self.major = int( a[ 0 ] )
+ if l > 1 :
+ self.minor = int( a[ 1 ] )
+ if l > 2 :
+ self.release = int( a[ 2 ] )
+ elif isinstance( s, int ) :
+ self.major = ( s & 0xFF000000 ) >> 24
+ self.minor = ( s & 0xFF0000 ) >> 16
+ self.release = s & 0xFFFF
+
+ def __str__( self ) :
+ return "%d.%d.%d"%( self.major, self.minor, self.release )
+
+ def __cmp__( self, other ) :
+ if not isinstance( other, version ) :
+ return NotImplemented
+ d = cmp( self.major, other.major )
+ if d != 0 :
+ return d
+ d = cmp( self.minor, other.minor )
+ if d != 0 :
+ return d
+ return cmp( self.release, other.release )
+
+
+#------------------------------------------------------------------------------------------------------------
+class vdb_error( Exception ) :
+ """Exception thrown by vdb-objects like mananger, schema, database, table, cursor, column
+
+ Args:
+ rc (int) : rc-code from vdb-library call
+ msg ( string ) : explanation of error
+ obj : object that caused the error ( manager, schema, database, table, cursor, column )
+ """
+ def __init__( self, rc, msg, obj ) :
+ super( vdb_error, self ).__init__( "%s.%s"%( obj.__class__.__name__, msg ) )
+ self.obj_name = obj.__class__.__name__
+ self.rc = rc
+
+
+#------------------------------------------------------------------------------------------------------------
+class KNamelist :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.KNamelistRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KNamelistRelease()", self )
+
+ def count( self ) :
+ n = c_int()
+ rc = self.__mgr.KNamelistCount( self.__ptr, byref( n ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KNamelistCount()", self )
+ return n.value
+
+ def to_list( self ) :
+ res = list()
+ for idx in xrange( 0, self.count() ) :
+ name = c_char_p()
+ rc = self.__mgr.KNamelistGet( self.__ptr, idx, byref( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KNamelistGet( %d )"%( idx ), self )
+ if PY3 :
+ res.append( name.value.decode( "utf-8" ) )
+ else :
+ res.append( name.value )
+ return res
+
+
+#------------------------------------------------------------------------------------------------------------
+def random_string( size = 12, chars = string.ascii_uppercase + string.ascii_lowercase + string.digits ) :
+ return ''.join( random.choice( chars ) for _ in xrange( size ) )
+
+def random_data( count, min_value, max_value ) :
+ res = list()
+ for _ in xrange( count ) :
+ res.append( random.randint( min_value, max_value ) )
+ return res
+
+#------------------------------------------------------------------------------------------------------------
+class typedecl( Structure ) :
+ _fields_ = [ ( "type_id", c_int ), ( "dim", c_int ) ]
+
+ def __str__( self ) :
+ return "( type_id=%d, dim=%d )"%( self.type_id, self.dim )
+
+class typedesc( Structure ) :
+ _fields_ = [ ( "bits", c_int ), ( "dim", c_int ), ( "domain", c_int ) ]
+
+ def __str__( self ) :
+ return "( bits=%d, dim=%d, domain=%d )"%( self.bits, self.dim, self.domain )
+
+
+#------------------------------------------------------------------------------------------------------------
+uint_xf = { 8 : c_ubyte, 16 : c_ushort, 32 : c_uint, 64 : c_ulonglong }
+int_xf = { 8 : c_byte, 16 : c_short, 32 : c_int, 64 : c_longlong }
+float_xf = { 32 : c_float, 64 : c_double }
+txt_xf = { 8 : c_char }
+
+if platform.system() == "Windows" :
+ type_xf = { TypeDomain.Bool : ( uint_xf, c_ubyte ),
+ TypeDomain.UInt : ( uint_xf, c_ubyte ),
+ TypeDomain.Int : ( int_xf, c_byte ),
+ TypeDomain.Float : ( float_xf, c_double ),
+ TypeDomain.Ascii : ( txt_xf, c_char ),
+ TypeDomain.Unicode : ( txt_xf, c_char ) }
+
+else :
+ type_xf = { TypeDomain.Bool.value : ( uint_xf, c_ubyte ),
+ TypeDomain.UInt.value : ( uint_xf, c_ubyte ),
+ TypeDomain.Int.value : ( int_xf, c_byte ),
+ TypeDomain.Float.value : ( float_xf, c_double ),
+ TypeDomain.Ascii.value : ( txt_xf, c_char ),
+ TypeDomain.Unicode.value : ( txt_xf, c_char ) }
+
+
+#------------------------------------------------------------------------------------------------------------
+class VColumn :
+ """representing a column of a vdb-cursor
+ """
+ def __init__( self, mgr, cur, id, name, tabname ) :
+ self.__mgr = mgr
+ self.__cur = cur
+ self.__id = id
+ p1 = name.find( '(' )
+ p2 = name.find( ')' )
+ if p1 > -1 and p2 > -1 :
+ self.cast = name[ p1 + 1 : p2 ]
+ self.name = name[ p2 + 1 : ]
+ else :
+ self.cast = ""
+ self.name = name
+ self.tabname = tabname
+ self.__tdec = typedecl( 0, 0 )
+ self.__tdes = typedesc( 0, 0, 0 )
+ self.column_type = None
+ self.min_value = None
+ self.max_value = None
+
+ def __str__( self ) :
+ return "%s.%s: (%d) %s %s"%( self.tabname, self.name, self.__id, self.tdec, self.tdes )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def _update( self ) :
+ rc = self.__mgr.VCursorDatatype( self.__cur._VCursor__ptr, self.__id, byref( self.__tdec ), byref( self.__tdes ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorDatatype( '%s.%s' )"%( self.tabname, self.name ), self )
+ ( dict, dflt ) = type_xf[ self.__tdes.domain ]
+ self.column_type = dict[ self.__tdes.bits ]
+ if self.column_type == None :
+ self.column_type == dflt
+
+ def domain( self ) :
+ return TypeDomain( self.__tdes.domain )
+
+ def bits( self ) :
+ return self.__tdes.bits
+
+ def dim( self ) :
+ return self.__tdes.dim
+
+ def Read( self, row ) :
+ """read values from a column
+ returns either a string or a list of integer, float, boolean values
+
+ Args:
+ row (longlong) : row to read from
+ """
+ if self.column_type == None :
+ raise vdb_error( 0, "read: undefined column-type", self )
+ row_id = c_longlong( row )
+ data = c_void_p()
+ row_len = c_int()
+ rc = self.__mgr.VCursorCellDataDirect( self.__cur._VCursor__ptr, row_id, self.__id, None, byref( data ), None, byref( row_len ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorCellDataDirect( '%s.%s', #%d )"%( self.tabname, self.name, row ), self )
+ if self.column_type == c_char :
+ return string_at( data, row_len.value )
+ else :
+ typed_ptr = cast( data, POINTER( self.column_type ) )
+ l = list()
+ for idx in xrange( 0, row_len.value ) :
+ l.append( typed_ptr[ idx ] )
+ return l
+
+ def __write_values( self, data ) :
+ if isinstance( data, list ) :
+ l = len( data )
+ t = self.column_type * l
+ arr = t()
+ idx = 0
+ for x in data :
+ arr[ idx ] = x
+ idx += 1
+ else :
+ l = 1
+ t = self.column_type * l
+ arr = t()
+ arr[ 0 ] = data
+
+ bits = c_int( self.__tdes.bits )
+ rc = self.__mgr.VCursorWrite( self.__cur._VCursor__ptr, self.__id, bits, arr, c_int( 0 ), c_int( l ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorWrite( %s.%s )"%( self.tabname, self.name ), self )
+
+ def __write_string( self, data ) :
+ p = create_string_buffer( to_bytes( data ) )
+ rc = self.__mgr.VCursorWrite( self.__cur._VCursor__ptr, self.__id, c_int( 8 ), p, c_int( 0 ), c_int( len( data ) ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorWrite( %s.%s, %s )"%( self.tabname, self.name, data ), self )
+
+ def write( self, data ) :
+ """write values to a column
+ raises vdb_error if error occurs
+
+ Args:
+ data ( list of values or string ) : data to be written
+ """
+ if self.column_type == None :
+ raise vdb_error( 0, "write: undefined column-type", self )
+ elif self.column_type == c_char :
+ self.__write_string( data )
+ else :
+ self.__write_values( data )
+
+ def write_rand( self, count = 1, max_value = 255, min_value = 0 ) :
+ if self.column_type == None :
+ raise vdb_error( 0, "write_rand: undefined column-type", self )
+ else :
+ dom = self.domain()
+ if dom == TypeDomain.Ascii or dom == TypeDomain.Unicode :
+ self.__write_string( random_string( count ) )
+ elif dom == TypeDomain.Bool :
+ self.__write_values( random_data( count, 0, 1 ) )
+ else :
+ self.__write_values( random_data( count, min_value, max_value ) )
+
+ def set_default( self, data ) :
+ if isinstance( data, list ) :
+ l = len( data )
+ t = self.column_type * l
+ arr = t()
+ idx = 0
+ for x in data :
+ arr[ idx ] = x
+ idx += 1
+ else :
+ l = 1
+ t = self.column_type * l
+ arr = t()
+ arr[ 0 ] = data
+
+ bits = c_int( self.__tdes.bits )
+ rc = self.__mgr.VCursorDefault( self.__cur._VCursor__ptr, self.__id, bits, arr, c_int( 0 ), c_int( l ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorDefault( %s.%s )"%( self.tabname, self.name ), self )
+
+ def default_string( self, data ) :
+ p = create_string_buffer( data )
+ rc = self.__mgr.VCursorDefault( self.__cur._VCursor__ptr, self.__id, c_int( 8 ), p, c_int( 0 ), c_int( len( data ) ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorDefault( %s.%s, %s )"%( self.tabname, self.name, data ), self )
+
+ def default( self, data ) :
+ if self.column_type == None :
+ raise vdb_error( 0, "default: undefined column-type", self )
+ if self.column_type == c_char :
+ return self.default_string( data )
+ else :
+ return self.default_values( data )
+
+ def row_range( self ) :
+ first = c_longlong()
+ count = c_longlong()
+ rc = self.__mgr.VCursorIdRange( self.__cur._VCursor__ptr, self.__id, byref( first ), byref( count ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorIdRange( '%s.%s' )"%( self.tabname, self.name ), self )
+ return ( first.value, count.value )
+
+ def range( self ) :
+ ( first, count ) = self.row_range()
+ return xrange( first, first + count )
+
+ def next_row( self, current_row ) :
+ res = c_longlong( 0 )
+ rc = self.__mgr.VCursorFindNextRowIdDirect( self.__cur._VCursor__ptr, self.__id, c_longlong( current_row ), byref( res ) )
+ if rc != 0 :
+ return None
+ else :
+ return res.value
+
+
+#------------------------------------------------------------------------------------------------------------
+class ReferenceObj :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ self.__mgr.ReferenceObj_Release( self.__ptr )
+
+ def __str__( self ) :
+ idx = self.Idx()
+ start, stop = self.IdRange()
+ seq_id = self.SeqId()
+ name = self.Name()
+ return "Idx\t%d\nIdRange\t%d..%d\nSeqId\t%s\nName\t%s"%( idx, start, stop, seq_id, name )
+
+ def Idx( self ) :
+ res = c_int()
+ rc = self.__mgr.ReferenceObj_Idx( self.__ptr, byref( res ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_Idx()", self )
+ return res.value
+
+ def IdRange( self ) :
+ start = c_longlong()
+ stop = c_longlong()
+ rc = self.__mgr.ReferenceObj_IdRange( self.__ptr, byref( start ), byref( stop ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_IdRange()", self )
+ return ( start.value, stop.value )
+
+ def Bin( self ) :
+ res = c_int()
+ rc = self.__mgr.ReferenceObj_Bin( self.__ptr, byref( res ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_Bin()", self )
+ return res.value
+
+ def SeqId( self ) :
+ res = c_char_p()
+ rc = self.__mgr.ReferenceObj_SeqId( self.__ptr, byref( res ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_SeqId()", self )
+ return res.value
+
+ def Name( self ) :
+ res = c_char_p()
+ rc = self.__mgr.ReferenceObj_Name( self.__ptr, byref( res ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_Name()", self )
+ return res.value
+
+ def SeqLength( self ) :
+ res = c_int()
+ rc = self.__mgr.ReferenceObj_SeqLength( self.__ptr, byref( res ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_SeqLength()", self )
+ return res.value
+
+ def Circular( self ) :
+ res = c_bool()
+ rc = self.__mgr.ReferenceObj_Circular( self.__ptr, byref( res ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_Circular()", self )
+ return res.value
+
+ def External( self ) :
+ res = c_bool()
+ rc = self.__mgr.ReferenceObj_External( self.__ptr, byref( res ), c_void_p( 0 ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_External()", self )
+ return res.value
+
+ def Read( self, offs, len ) :
+ buffer = create_string_buffer( len )
+ written = c_int()
+ rc = self.__mgr.ReferenceObj_Read( self.__ptr, c_int( offs ), c_int( len ), buffer, byref( written ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_Read( %d.%d )" % ( offs, len ), self )
+ if PY3 :
+ return buffer.value.decode( "utf-8" )
+ return buffer.value
+
+ def GetIdCount( self, row_id ) :
+ res = c_int()
+ rc = self.__mgr.ReferenceObj_GetIdCount( self.__ptr, c_longlong( row_id ), byref( res ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceObj_GetIdCount( %d )" % row_id, self )
+ return res.value
+
+
+#------------------------------------------------------------------------------------------------------------
+class ReferenceList :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.ReferenceList_Release( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceList_Release()", self )
+
+ def count( self ) :
+ res = c_int()
+ rc = self.__mgr.ReferenceList_Count( self.__ptr, byref( res ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceList_Count()", self )
+ return res.value
+
+ def find( self, name ) :
+ ptr = c_void_p()
+ rc = self.__mgr.ReferenceList_Find( self.__ptr, byref( ptr ), to_char_p( name ), c_int( len( name ) ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceList_Find( '%s' )" % name, self )
+ return ReferenceObj( self.__mgr, ptr )
+
+ def get( self, idx ) :
+ ptr = c_void_p()
+ rc = self.mgr.ReferenceList_Get( self.__ptr, byref( ptr ), c_int( idx ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceList_Get( %d )" % idx, self )
+ return ReferenceObj( self.__mgr, ptr )
+
+
+#------------------------------------------------------------------------------------------------------------
+def read_row( cols, row_id ) :
+ res = {}
+ try :
+ for ( name, column ) in cols.iteritems() :
+ res[ name ] = column.Read( row_id )
+ except AttributeError :
+ for ( name, column ) in cols.items() :
+ res[ name ] = column.Read( row_id )
+ return res
+
+
+#------------------------------------------------------------------------------------------------------------
+def first_none_static_col( cols ) :
+ res = None
+ try :
+ v = cols.itervalues()
+ except AttributeError :
+ v = cols.values()
+ for c in v :
+ rr = c.row_range()
+ if res == None and rr[ 1 ] > 0 :
+ res = c
+ return res
+
+
+#------------------------------------------------------------------------------------------------------------
+def row_gen( cols, row_range = None ) :
+ if isinstance( cols, dict ) :
+ first_col = first_none_static_col( cols )
+ if first_col != None :
+ if row_range == None :
+ row_id = 1
+ while row_id != None :
+ yield read_row( cols, row_id )
+ row_id = first_col.next_row( row_id + 1 )
+ else :
+ range_idx = 0
+ try :
+ row_id = row_range[ range_idx ]
+ except :
+ row_id = None
+ while row_id != None :
+ yield read_row( cols, row_id )
+ range_idx += 1
+ try :
+ row_id = first_col.next_row( row_range[ range_idx ] )
+ except :
+ row_id = None
+ else :
+ if row_range == None :
+ row_id = 1
+ while row_id != None :
+ yield cols.Read( row_id )
+ row_id = cols.next_row( row_id + 1 )
+ else :
+ range_idx = 0
+ try :
+ row_id = row_range[ range_idx ]
+ except :
+ row_id = None
+ while row_id != None :
+ yield cols.Read( row_id )
+ range_idx += 1
+ try :
+ row_id = cols.next_row( row_range[ range_idx ] )
+ except :
+ row_id = None
+
+
+#------------------------------------------------------------------------------------------------------------
+class VCursor :
+ def __init__( self, tab, ptr ) :
+ self.__mgr = tab._VTable__mgr
+ self.__tab = tab
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.VCursorRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorRelease( '%s' )"%( self.__tab.name ), self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def OpenColumns( self, col_names ) :
+ if isinstance( col_names, list ) :
+ res = {}
+ for name in col_names :
+ res[ name ] = self.AddColumn( name )
+ self.Open()
+ try :
+ v = res.itervalues()
+ except AttributeError :
+ v = res.values()
+ for c in v :
+ c._update()
+ return res
+ elif isinstance( col_names, str ) :
+ c = self.AddColumn( col_names )
+ self.Open()
+ c._update()
+ return c
+ else :
+ raise vdb_error( 0, "cursor.open( x ) x is not list or string", self )
+
+ def AddColumn( self, name ) :
+ idx = c_int()
+ rc = self.__mgr.VCursorAddColumn( self.__ptr, byref( idx ), to_char_p( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorAddColumn( %s.%s )"%( self.__tab.name, name ), self )
+ return VColumn( self.__mgr, self, idx.value, name, self.__tab._VTable__name )
+
+ def Open( self ) :
+ rc = self.__mgr.VCursorOpen( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorOpen( '%s' )"%( self.__tab.name ), self )
+
+ def Commit( self ) :
+ rc = self.__mgr.VCursorCommit( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorCommit( %s )"%( self.__tab.name ), self )
+
+ def OpenRow( self ) :
+ rc = self.__mgr.VCursorOpenRow( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorOpenRow( '%s' )"%( self.__tab.name ), self )
+
+ def CommitRow( self ) :
+ rc = self.__mgr.VCursorCommitRow( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorCommitRow( %s )"%( self.__tab.name ), self )
+
+ def RepeatRow( self, count ) :
+ rc = self.__mgr.VCursorRepeatRow( self.__ptr, count )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorRepeatRow( %s, %d )"%( self.__tab.name, count ), self )
+
+ def CloseRow( self ) :
+ rc = self.__mgr.VCursorCloseRow( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorCloseRow( '%s' )"%( self.__tab.name ), self )
+
+ def FlushPage( self ) :
+ rc = self.__mgr.VCursorFlushPage( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorFlushPage( %s )"%( self.__tab.name ), self )
+
+ def RowId( self ) :
+ row_id = c_longlong()
+ rc = self.__mgr.VCursorRowId( self.__ptr, byref( row_id ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VCursorRowId( '%s' )"%( self.__tab.name ), self )
+ return row_id.value
+
+ def ReferenceList( self ) :
+ reflist_ptr = c_void_p()
+ rc = self.__mgr.ReferenceList_MakeCursor( byref( reflist_ptr ), self.__ptr, c_int( 0 ), c_char_p( 0 ), c_int( 0 ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceList_MakeCursor()", self )
+ return ReferenceList( self.__mgr, reflist_ptr )
+
+
+def max_colname_len( cols, colnames ) :
+ res = 0
+ if colnames == None :
+ try :
+ for ( name, column ) in cols.iteritems() :
+ l = len( name )
+ if l > res :
+ res = l
+ except AttributeError :
+ for ( name, column ) in cols.items() :
+ l = len( name )
+ if l > res :
+ res = l
+ else :
+ for name in colnames :
+ l = len( name )
+ if l > res :
+ res = l
+ return res
+
+
+#------------------------------------------------------------------------------------------------------------
+# cols ....... dictionay : key = colname, value = column-object
+# rowrange ... a xrange - iterator ( None: all rows )
+# colnames ... a list of string ( None: all columns )
+# prefix ..... a string printed at the beginning of each line
+def print_cols( cols, rowrange, colnames = None, prefix = "" ) :
+ if cols != None :
+ if rowrange == None :
+ try :
+ column = cols.itervalues().next()
+ except AttributeError :
+ column = list( cols.values() )[ 0 ]
+ rr = column.range()
+ else :
+ rr = rowrange
+ w = max_colname_len( cols, colnames )
+ for row_id in rr :
+ if colnames == None :
+ try :
+ for ( name, column ) in sorted( cols.iteritems() ) :
+ print( '{0}{1:<{width}}.{2} : {3}'.format( prefix, name, row_id, column.Read( row_id ), width = w ) )
+ except AttributeError :
+ for ( name, column ) in sorted( cols.items() ) :
+ print( '{0}{1:<{width}}.{2} : {3}'.format( prefix, name, row_id, column.Read( row_id ), width = w ) )
+ else :
+ for name in colnames :
+ column = cols[ name ]
+ if column != None :
+ print( '{0}{1:<{width}}.{2} : {3}'.format( prefix, name, row_id, column.Read( row_id ), width = w ) )
+ print( "%s."%( prefix ) )
+
+
+#------------------------------------------------------------------------------------------------------------
+class KIndex :
+ def __init__( self, mgr, ptr, name ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+ self.__name = name
+
+ def __del__( self ) :
+ rc = self.__mgr.KIndexRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexRelease( '%s' )"%( self.__name ), self )
+
+ def Version( self ) :
+ vers = c_int()
+ rc = self.__mgr.KIndexVersion( self.__ptr, byref( vers ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexVersion( '%s' )"%( self.__name ), self )
+ return version( vers.value )
+
+ def Type( self ) :
+ type = c_int()
+ rc = self.__mgr.KIndexType( self.__ptr, byref( type ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexType( '%s' )"%( self.__name ), self )
+ return IndexType( type.value )
+
+ def Locked( self ) :
+ return self.__mgr.KIndexLocked( self.__ptr )
+
+ def FindText( self, key ) :
+ start_id = c_longlong()
+ id_count = c_longlong()
+ rc = self.__mgr.KIndexFindText( self.__ptr, to_char_p( key ), byref( start_id ), by_ref( id_count ), c_void_p( 0 ), c_void_p( 0 ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexFindText( %s, %s )"%( self.__name, key ), self )
+ return ( start_id.value, id_count.value )
+
+ def ProjectText( self, row_id, bufsize = 1024 ) :
+ buffer = create_string_buffer( bufsize )
+ start_id = c_longlong()
+ id_count = c_longlong()
+ num_writ = c_int( 0 )
+ rc = self.__mgr.KIndexProjectText( self.__ptr, c_longlong( row_id ), byref( start_id ), byref( id_count ), buffer, c_int( bufsize ), byref( num_writ ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexProjectText( %s, %d )"%( self.__name, row_id ), self )
+ if PY3 :
+ return ( buffer.value.decode( "utf-8" ), start_id.value, id_count.value )
+ return ( buffer.value, start_id.value, id_count.value )
+
+ def Commit( self ) :
+ rc = self.__mgr.KIndexCommit( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexCommit( %s )"%( self.__name ), self )
+
+ def InsertText( self, unique, key, value ) :
+ rc = self.__mgr.KIndexInsertText( self.__ptr, c_bool( unique ), to_char_p( key ), c_longlong( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexInsertText( %s, k=%s, v=%d )"%( self.__name, key, value ), self )
+
+ def DeleteText( self, key ) :
+ rc = self.__mgr.KIndexDeleteText( self.__ptr, to_char_p( key ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexDeleteText( %s, k=%s )"%( self.__name, key ), self )
+
+ def InsertU64( self, unique, key_start, keylen, value_start, value_len ) :
+ rc = self.__mgr.KIndexInsertU64( self.__ptr, c_bool( unique ), c_longlong( key_start ), c_longlong( key_len ), c_longlong( value_start ), c_longlong( value_len ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexInsertU64( %s, k=%d.%d, v=%d.%d )"%( self.__name, key_start, key_len, value_start, value_len ), self )
+
+ def DeleteU64( self, key ) :
+ rc = self.__mgr.KIndexDeleteU64( self.__ptr, c_longlong( key ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KIndexDeleteU64( %s, k=%d )"%( self.__name, key ), self )
+
+
+#------------------------------------------------------------------------------------------------------------
+class KMDataNode :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.KMDataNodeRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeRelease()", self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def OpenNode( self, path, open_mode = OpenMode.Read ) :
+ node = c_void_p()
+ if open_mode == OpenMode.Write :
+ rc = self.__mgr.KMDataNodeOpenNodeUpdate( self.__ptr, byref( node ), to_char_p( path ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeOpenNodeUpdate( %s )"%( path ), self )
+ else :
+ rc = self.__mgr.KMDataNodeOpenNodeRead( self.__ptr, byref( node ), to_char_p( path ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeOpenNodeRead( %s )"%( path ), self )
+ return KMDataNode( self.__mgr, node )
+
+ def ByteOrder( self ) :
+ order = c_bool()
+ rc = self.__mgr.KMDataNodeByteOrder( self.__ptr, byref( order ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeByteOrder()", self )
+ return order.value
+
+ def ListChildren( self ) :
+ l = c_void_p()
+ rc = self.__mgr.KMDataNodeListChildren( self.__ptr, byref( l ) )
+ if rc == 0 :
+ return KNamelist( self.__mgr, l ).to_list()
+ return []
+
+ def ListAttr( self ) :
+ l = c_void_p()
+ rc = self.__mgr.KMDataNodeListAttr( self.__ptr, byref( l ) )
+ if rc == 0 :
+ return KNamelist( self.__mgr, l ).to_list()
+ return []
+
+ def size( self ) :
+ buffer = create_string_buffer( 16 )
+ num_read = c_size_t()
+ remaining = c_size_t()
+ rc = self.__mgr.KMDataNodeRead( self.__ptr, c_size_t( 0 ), buffer, c_size_t( 16 ), byref( num_read ), byref( remaining ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeRead()", self )
+ return num_read.value + remaining.value
+
+ def to_read( self, req_len ) :
+ ns = self.size()
+ if req_len > 0 :
+ return min( req_len, ns )
+ return ns
+
+ def Read( self, buffer_size, to_read, offset ) :
+ buffer = create_string_buffer( int( buffer_size ) )
+ num_read = c_size_t()
+ remaining = c_size_t()
+ rc = self.__mgr.KMDataNodeRead( self.__ptr, c_size_t( offset ), buffer, c_size_t( to_read ), byref( num_read ), byref( remaining ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeRead( offset = %d )"%( offset ), self )
+ if PY3 :
+ return buffer.value.decode( "utf-8" )
+ return buffer.value
+
+ def as_string( self, req_len = 0, offset = 0 ) :
+ n_bytes = self.to_read( req_len )
+ if n_bytes < 1 :
+ return ""
+ return self.Read( n_bytes, n_bytes, offset )
+
+ def read_buffer_as( self, t = c_ubyte, align = 1, req_len = 0, offset = 0 ) :
+ n_bytes = self.to_read( req_len )
+ if n_bytes < 1 :
+ return []
+ n_values = n_bytes
+ if align > 1 :
+ n_values = n_bytes / align
+ if n_bytes > ( n_values * align ) :
+ n_values += 1
+ buffer = self.Read( n_values * align, n_bytes, offset )
+ ptr = cast( buffer, POINTER( t ) )
+ l = list()
+ for idx in xrange( 0, int( n_values ) ) :
+ l.append( ptr[ idx ] )
+ return l
+
+ def as_uint8( self, req_len = 0, offset = 0 ) :
+ return self.read_buffer_as( c_ubyte, 1, req_len, offset )
+
+ def as_int8( self, req_len = 0, offset = 0 ) :
+ return self.read_buffer_as( c_byte, 1, req_len, offset )
+
+ def as_uint16( self, req_len = 0, offset = 0 ) :
+ return self.read_buffer_as( c_ushort, 2, req_len, offset )
+
+ def as_int16( self, req_len = 0, offset = 0 ) :
+ return self.read_buffer_as( c_short, 2, req_len, offset )
+
+ def as_uint32( self, req_len = 0, offset = 0 ) :
+ return self.read_buffer_as( c_uint, 4, req_len, offset )
+
+ def as_int32( self, req_len = 0, offset = 0 ) :
+ return self.read_buffer_as( c_int, 4, req_len, offset )
+
+ def as_uint64( self, req_len = 0, offset = 0 ) :
+ return self.read_buffer_as( c_ulonglong, 8, req_len, offset )
+
+ def as_int64( self, req_len = 0, offset = 0 ) :
+ return self.read_buffer_as( c_longlong, 8, req_len, offset )
+
+ def write_string( self, data, append = False ) :
+ p = create_string_buffer( data )
+ if append :
+ rc = self.__mgr.KMDataNodeAppend( self.__ptr, p, c_size_t( len( data ) ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeAppend()", self )
+ else :
+ rc = self.__mgr.KMDataNodeWrite( self.__ptr, p, c_size_t( len( data ) ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeWrite()", self )
+
+ def write_values( self, data, ct, ct_size, append ) :
+ if isinstance( data, list ) :
+ l = len( data )
+ t = ct * l
+ arr = t()
+ idx = 0
+ for x in data :
+ arr[ idx ] = x
+ idx += 1
+ else :
+ l = 1
+ t = ct * l
+ arr = t()
+ arr[ 0 ] = data
+ if append :
+ rc = self.__mgr.KMDataNodeAppend( self.__ptr, arr, c_size_t( l * ct_size ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeAppend()", self )
+ else :
+ rc = self.__mgr.KMDataNodeWrite( self.__ptr, arr, c_size_t( l * ct_size ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMDataNodeWrite()", self )
+
+ def write_uint8( self, data, append = False ) :
+ self.write_values( data, c_ubyte, 1, append )
+
+ def write_int8( self, data, append = False ) :
+ self.write_values( data, c_byte, 1, append )
+
+ def write_uint16( self, data, append = False ) :
+ self.write_values( data, c_ushort, 2, append )
+
+ def write_int16( self, data, append = False ) :
+ self.write_values( data, c_short, 2, append )
+
+ def write_uint32( self, data, append = False ) :
+ self.write_values( data, c_uint, 4, append )
+
+ def write_int32( self, data, append = False ) :
+ self.write_values( data, c_int, 4, append )
+
+ def write_uint64( self, data, append = False ) :
+ self.write_values( data, c_ulonglong, 8, append )
+
+ def write_int64( self, data, append = False ) :
+ self.write_values( data, c_longlong, 8, append )
+
+
+#------------------------------------------------------------------------------------------------------------
+class KMetadata :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.KMetadataRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataRelease()", self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def Version( self ) :
+ vers = c_int()
+ rc = self.__mgr.KMetadataVersion( self.__ptr, byref( vers ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataVersion()", self )
+ return version( vers.value )
+
+ def ByteOrder( self ) :
+ order = c_bool()
+ rc = self.__mgr.KMetadataByteOrder( self.__ptr, byref( order ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataByteOrder()", self )
+ return order.value
+
+ def Revision( self ) :
+ rev = c_int()
+ rc = self.__mgr.KMetadataRevision( self.__ptr, byref( rev ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataRevision()", self )
+ return rev.value
+
+ def MaxRevision( self ) :
+ rev = c_int()
+ rc = self.__mgr.KMetadataMaxRevision( self.__ptr, byref( rev ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataMaxRevision()", self )
+ return rev.value
+
+ def Commit( self ) :
+ rc = self.__mgr.KMetadataCommit( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataCommit()", self )
+
+ def Freeze( self ) :
+ rc = self.__mgr.KMetadataFreeze( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataFreeze()", self )
+
+ def OpenRevision( self, rev_nr ) :
+ k_meta = c_void_p()
+ rc = self.__mgr.KMetadataOpenRevision( self.__ptr, byref( k_meta ), c_int( rev_nr ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataOpenRevision( %d )"%rev_nr, self )
+ return KMetadata( self.__mgr, k_meta )
+
+ def GetSequence( self, name ) :
+ value = c_longlong()
+ rc = self.__mgr.KMetadataGetSequence( self.__ptr, to_char_p( name ), byref( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataGetSequence( %s )"%name, self )
+ return value.value
+
+ def SetSequence( self, name, value ) :
+ rc = self.__mgr.KMetadataSetSequence( self.__ptr, to_char_p( name ), c_longlong( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataSetSequence( %s, %d )"%( name, value ), self )
+
+ def NextSequence( self, name ) :
+ value = c_longlong()
+ rc = self.__mgr.KMetadataNextSequence( self.__ptr, to_char_p( name ), byref( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataNextSequence( %s )"%( name ), self )
+ return value.value
+
+ def OpenNode( self, path, open_mode = OpenMode.Read ) :
+ ptr = c_void_p()
+ if open_mode == OpenMode.Write :
+ rc = self.__mgr.KMetadataOpenNodeUpdate( self.__ptr, byref( ptr ), to_char_p( path ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataOpenNodeUpdate( %s )"%( path ), self )
+ else :
+ rc = self.__mgr.KMetadataOpenNodeRead( self.__ptr, byref( ptr ), to_char_p( path ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KMetadataOpenNodeRead( %s )"%( path ), self )
+ return KMDataNode( self.__mgr, ptr )
+
+
+#------------------------------------------------------------------------------------------------------------
+class VTable :
+ def __init__( self, mgr, ptr, name ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+ self.__kdb_ptr = None
+ self.__name = name
+ self.p_cur = None
+ self.p_cols = None
+
+ def __del__( self ) :
+ rc = self.__mgr.VTableRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VTableRelease( %s )"%( self.__name ), self )
+ if self.__kdb_ptr != None :
+ rc = self.__mgr.KTableRelease( self.__kdb_ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KTableRelease( %s )"%( self.__name ), self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def Name( self ) :
+ return self.__name
+
+ def ListCol( self ) :
+ l = c_void_p()
+ rc = self.__mgr.VTableListCol( self.__ptr, byref( l ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VTableListCol( %s )"%( self.__name ), self )
+ return KNamelist( self.__mgr, l ).to_list()
+
+ def __OpenKTableRead__( self ) :
+ k_table = c_void_p()
+ rc = self.__mgr.VTableOpenKTableRead( self.__ptr, byref( k_table ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VTableOpenKTableRead( '%s' )"%( self.__name ), self )
+ self.__kdb_ptr = k_table
+
+ def __OpenKTableUpdate__( self ) :
+ k_table = c_void_p()
+ rc = self.__mgr.VTableOpenKTableUpdate( self.__ptr, byref( k_table ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VTableOpenKTableUpdate( '%s' )"%( self.__name ), self )
+ self.__kdb_ptr = k_table
+
+ def ListIdx( self ) :
+ if self.__kdb_ptr == None :
+ self.__OpenKTableRead__()
+ l = c_void_p()
+ rc = self.__mgr.KTableListIdx( self.__kdb_ptr, byref( l ) )
+ if rc == 0 :
+ return KNamelist( self.__mgr, l ).to_list()
+ return []
+
+ def OpenIndexRead( self, name ) :
+ if self.__kdb_ptr == None :
+ self.__OpenKTableUpdate__()
+ k_idx = c_void_p()
+ rc = self.__mgr.KTableOpenIndexRead( self.__kdb_ptr, byref( k_idx ), to_char_p( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KTableOpenIndexRead( '%s' )"%( self.__name ), self )
+ return KIndex( self.__mgr, k_idx, name )
+
+ def CreateIndex( self, name, idx_type = IndexType.Text, create_mode = CreateMode.Init ) :
+ if self.__kdb_ptr == None :
+ self.__OpenKTableUpdate__()
+ k_idx = c_void_p()
+ rc = self.__mgr.KTableCreateIndex( self.__kdb_ptr, byref( k_idx ), c_int( idx_type.value ), c_int( create_mode.value ), to_char_p( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KTableCreateIndex( '%s' )"%( self.__name ), self )
+ return KIndex( self.__mgr, k_idx, name )
+
+ def OpenMetadata( self, open_mode = OpenMode.Read ) :
+ if self.__kdb_ptr == None :
+ self.__OpenKTableRead__()
+ k_meta = c_void_p()
+ if open_mode == OpenMode.Write :
+ rc = self.__mgr.KTableOpenMetadataUpdate( self.__kdb_ptr, byref( k_meta ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KTableOpenMetadataUpdate( '%s' )"%( self.__name ), self )
+ else :
+ rc = self.__mgr.KTableOpenMetadataRead( self.__kdb_ptr, byref( k_meta ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KTableOpenMetadataRead( '%s' )"%( self.__name ), self )
+ return KMetadata( self.__mgr, k_meta )
+
+ def CreateCursor( self, open_mode = OpenMode.Read, ins_mode = CursorMode.Insert ) :
+ c = c_void_p()
+ if open_mode == OpenMode.Write :
+ rc = self.__mgr.VTableCreateCursorWrite( self.__ptr, byref( c ), c_int( ins_mode.value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VTableCreateCursorWrite( %s )"%( self.__name ), self )
+ else :
+ rc = self.__mgr.VTableCreateCursorRead( self.__ptr, byref( c ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VTableCreateCursorRead( %s )"%( self.__name ), self )
+ return VCursor( self, c )
+
+ def CreateCachedCursorRead( self, cache_size ) :
+ c = c_void_p()
+ rc = self.__mgr.VTableCreateCachedCursorRead( self.__ptr, byref( c ), c_longlong( cache_size ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VTableCreateCachedCursorRead( %s, %d )"%( self.__name, cache_size ), self )
+ return VCursor( self, c )
+
+ def Locked( self ) :
+ return self.__mgr.VTableLocked( self.__ptr )
+
+ def ReferenceList( self, options = 2 ) :
+ ptr = c_void_p()
+ rc = self.__mgr.ReferenceList_MakeTable( byref( ptr ), self.__ptr, c_int( options ), c_int( 0 ), c_char_p( 0 ), c_int( 0 ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceList_MakeTable( '%s' )"%( self.__name ), self )
+ return ReferenceList( self.__mgr, ptr )
+
+ def print_rows( self, colrange = None, colnames = None, prefix = "" ) :
+ if self.p_cur == None :
+ self.p_cur = self.CreateCursor()
+ if self.p_cur != None :
+ if self.p_cols == None :
+ if colnames == None :
+ self.p_cols = self.p_cur.OpenColumns( self.ListCol() )
+ else :
+ self.p_cols = self.p_cur.OpenColumns( colnames )
+ elif colnames != None :
+ more_to_open = 0
+ for name in colnames :
+ if self.p_cols[ name ] == None :
+ more_to_open += 1
+ if more_to_open > 0 :
+ self.p_cur = None
+ self.p_cols = None
+ self.p_cur = self.CreateCursor()
+ if self.p_cur != None :
+ self.p_cols = self.p_cur.OpenColumns( colnames )
+ if self.p_cols != None :
+ print_cols( self.p_cols, colrange, colnames, prefix )
+
+
+#------------------------------------------------------------------------------------------------------------
+class VDatabase :
+ def __init__( self, mgr, ptr, name ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+ self.__name = name
+
+ def __del__( self ) :
+ rc = self.__mgr.VDatabaseRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseRelease( %s )"%( self.__name ), self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def Name( self ) :
+ return self.__name
+
+ def ListTbl( self ) :
+ l = c_void_p()
+ rc = self.__mgr.VDatabaseListTbl( self.__ptr, byref( l ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseListTbl( %s )"%( self.__name ), self )
+ return KNamelist( self.__mgr, l ).to_list()
+
+ def ListDB( self ) :
+ l = c_void_p()
+ rc = self.__mgr.VDatabaseListDB( self.__ptr, byref( l ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseListDB( %s )"%( self.__name ), self )
+ return KNamelist( self.__mgr, l ).to_list()
+
+ def OpenTable( self, name, mode = OpenMode.Read ) :
+ f = self.__mgr.VDatabaseOpenTableUpdate if mode == OpenMode.Write else self.__mgr.VDatabaseOpenTableRead
+ tab = c_void_p()
+ rc = f( self.__ptr, byref( tab ), to_char_p( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseOpenTable( %s.%s )"%( self.__name, name ), self )
+ return VTable( self.__mgr, tab, name )
+
+ def OpenDB( self, name, mode = OpenMode.Read ) :
+ f = self.__mgr.VDatabaseOpenDBUpdate if mode == OpenMode.Write else self.__mgr.VDatabaseOpenDBRead
+ db = c_void_p()
+ rc = f( self.__ptr, byref( db ), to_char_p( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseOpenDB( %s.%s )"%( self.__name, name ), self )
+ return VDatabase( self.__mgr, db, name )
+
+ def CreateTable( self, spec, name = None, mode = CreateMode.Init ) :
+ tab = c_void_p()
+ p = to_char_p( name if name != None else spec )
+ rc = self.__mgr.VDatabaseCreateTable( self.__ptr, byref( tab ), to_char_p( spec ), c_int( mode.value ), p )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseCreateTable( %s.%s )"%( self.__name, spec ), self )
+ return VTable( self.__mgr, tab, p.value )
+
+ def CreateDB( self, spec, name = None, mode = CreateMode.Init ) :
+ new_db = c_void_p()
+ p = to_char_p( name if name != None else spec )
+ rc = self.__mgr.VDatabaseCreateDB( self.__ptr, byref( new_db ), to_char_p( spec ), c_int( mode.value ), p )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseCreateDB( %s.%s )"%( self.__name, spec ), self )
+ return VDatabase( self.__mgr, new_db, p.value )
+
+ def DropTable( self, name ) :
+ rc = self.__mgr.VDatabaseDropTable( self.__ptr, to_char_p( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseDropTable( %s.%s )"%( self.__name, name ), self )
+
+ def DropDB( self, name ) :
+ rc = self.__mgr.VDatabaseDropDB( self.__ptr, to_char_p( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseDropDB( %s.%s )"%( self.__name, name ), self )
+
+ def Lock( self, name, what ) :
+ rc = self.__mgr.VDatabaseLock( self.__ptr, c_int( what ), to_char_p( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseLock( %s.%s )"%( self.__name, name ), self )
+
+ def UnLock( self, name, what ) :
+ rc = self.__mgr.VDatabaseUnlock( self.__ptr, c_int( what ), to_char_p( name ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VDatabaseUnlock( %s.%s )"%( self.__name, name ), self )
+
+ def Locked( self ) :
+ return self.__mgr.VDatabaseLocked( self.__ptr )
+
+ def ReferenceList( self, options = 2 ) :
+ ptr = c_void_p()
+ rc = self.__mgr.ReferenceList_MakeDatabase( byref( ptr ), self.__ptr, c_int( options ), c_int( 0 ), c_char_p( 0 ), c_int( 0 ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "ReferenceList_MakeDatabase( %s )"%( self.__name ), self )
+ return ReferenceList( self.__mgr, ptr )
+
+
+#------------------------------------------------------------------------------------------------------------
+class VSchema :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.VSchemaRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VSchemaRelease()", self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def ParseText( self, schema_txt ) :
+ txt = to_char_p( schema_txt )
+ l = len( schema_txt )
+ rc = self.__mgr.VSchemaParseText( self.__ptr, c_char_p( 0 ), txt, c_int( l ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VSchemaParseText()", self )
+
+ def ParseFile( self, schema_file ) :
+ rc = self.__mgr.VSchemaParseFile( self.__ptr, to_char_p( schema_file ) )
+ if rc != 0 :
+ self.mgr.raise_rc( rc, "VSchemaParseFile( '%s' )"%schema_file, self )
+
+ def AddIncludePath( self, path ) :
+ rc = self.__mgr.VSchemaAddIncludePath( self.__ptr, to_char_p( path ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VSchemaAddIncludePath( '%s' )"%path, self )
+
+
+#------------------------------------------------------------------------------------------------------------
+class KRepository :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.KRepositoryRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositoryRelease()", self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def __str__( self ) :
+ lst = [ "'%s'"%( self.Name() ) ]
+ lst.append( "%s"%( RepoCat( self.Category() ) ) )
+ lst.append( "%s"%( RepoSubCat( self.SubCategory() ) ) )
+ lst.append( "display: %s"%( self.DisplayName() ) )
+ lst.append( "root: %s"%( self.Root() ) )
+ lst.append( "resolver: %s"%( self.Resolver() ) )
+ lst.append( "disabled: %r"%( self.Disabled() ) )
+ lst.append( "cached: %r"%( self.CacheEnabled() ) )
+ return ", ".join( lst )
+
+ def AddRef ( self ) :
+ rc = self.__mgr.KRepositoryAddRef( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositoryAddRef()", self )
+
+ def Category( self ) :
+ res = self.__mgr.KRepositoryCategory( self.__ptr )
+ return res;
+
+ def SubCategory( self ) :
+ return self.__mgr.KRepositorySubCategory( self.__ptr )
+
+ def Name( self ) :
+ p = create_string_buffer( 1024 )
+ n = c_int( 0 )
+ rc = self.__mgr.KRepositoryName( self.__ptr, p, sizeof( p ), byref( n ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositoryName()", self )
+ if PY3 :
+ return p.value.decode( "utf-8" )
+ return p.value
+
+ def DisplayName( self ) :
+ p = create_string_buffer( 1024 )
+ n = c_int( 0 )
+ rc = self.__mgr.KRepositoryDisplayName( self.__ptr, p, sizeof( p ), byref( n ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositoryDisplayName()", self )
+ if PY3 :
+ return p.value.decode( "utf-8" )
+ return p.value
+
+ def Root( self ) :
+ p = create_string_buffer( 1024 )
+ n = c_int( 0 )
+ rc = self.__mgr.KRepositoryRoot( self.__ptr, p, sizeof( p ), byref( n ) )
+ if rc != 0 :
+ return ""
+ if PY3 :
+ return p.value.decode( "utf-8" )
+ return p.value
+
+ def SetRoot( self, root ) :
+ rc = self.__mgr.KRepositorySetRoot( self.__ptr, to_char_p( root ), c_int( len( root ) ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositorySetRoot( '%s' )"%( root ), self )
+
+ def Resolver( self ) :
+ p = create_string_buffer( 1024 )
+ n = c_int( 0 )
+ rc = self.__mgr.KRepositoryResolver( self.__ptr, p, sizeof( p ), byref( n ) )
+ if rc != 0 :
+ return ""
+ if PY3 :
+ return p.value.decode( "utf-8" )
+ return p.value
+
+ def Disabled( self ) :
+ return self.__mgr.KRepositoryDisabled( self.__ptr )
+
+ def SetDisabled( self, value ) :
+ rc = self.__mgr.KRepositorySetDisabled( self.__ptr, c_bool( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositorySetDisabled()", self )
+
+ def CacheEnabled( self ) :
+ return self.__mgr.KRepositoryCacheEnabled( self.__ptr )
+
+
+#------------------------------------------------------------------------------------------------------------
+class KRepositoryMgr :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.KRepositoryMgrRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositoryMgrRelease()", self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def HasRemoteAccess( self ) :
+ return self.__mgr.KRepositoryMgrHasRemoteAccess( self.__ptr );
+
+ def __vector_2_list__( self, v ) :
+ res = list()
+ for i in xrange( v.start, v.start + v.len ) :
+ r = KRepository( self.__mgr, self.__mgr.VectorGet( byref( v ), c_int( i ) ) )
+ r.AddRef()
+ res.append( r )
+ self.__mgr.KRepositoryVectorWhack( byref( v ) )
+ return res
+
+ def UserRepositories( self ) :
+ v = vdb_vector()
+ rc = self.__mgr.KRepositoryMgrUserRepositories( self.__ptr, byref( v ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositoryMgrUserRepositories()", self )
+ return self.__vector_2_list__( v )
+
+ def SiteRepositories( self ) :
+ v = vdb_vector()
+ rc = self.__mgr.KRepositoryMgrSiteRepositories( self.__ptr, byref( v ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositoryMgrSiteRepositories()", self )
+ return self.__vector_2_list__( v )
+
+ def RemoteRepositories( self ) :
+ v = vdb_vector()
+ rc = self.__mgr.KRepositoryMgrRemoteRepositories( self.__ptr, byref( v ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KRepositoryMgrRemoteRepositories()", self )
+ return self.__vector_2_list__( v )
+
+ def CategoryDisabled( self, cat ) :
+ return self.__mgr.KRepositoryMgrCategoryDisabled( self.__ptr, c_int( cat ) )
+
+ def CategorySetDisabled( self, cat, value ) :
+ return self.__mgr.KRepositoryMgrCategorySetDisabled( self.__ptr, c_int( cat ), c_bool( value ) )
+
+ def AllRepos( self ) :
+ return [ self.UserRepositories(), self.SiteRepositories(), self.RemoteRepositories() ]
+
+#------------------------------------------------------------------------------------------------------------
+class KConfig :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.KConfigRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigRelease()", self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def Commit( self ) :
+ rc = self.__mgr.KConfigCommit( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigCommit()", self )
+ return rc
+
+ def ReadBool( self, path ) :
+ value = c_bool()
+ rc = self.__mgr.KConfigReadBool( self.__ptr, to_char_p( path ), byref( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigReadBool( '%s' )"%( path ), self )
+ return value.value
+
+ def WriteBool( self, path, value ) :
+ rc = self.__mgr.KConfigWriteBool( self.__ptr, to_char_p( path ), c_bool( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigWriteBool( '%s', %s )"%( path, value ), self )
+ return rc
+
+ def ReadString( self, path ) :
+ s = vdb_string( None, 0, 0 )
+ sp = pointer( s )
+ rc = self.__mgr.KConfigReadString( self.__ptr, to_char_p( path ), byref( sp ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigReadString( '%s' )"%( path ), self )
+ res = sp.contents.addr
+ self.__mgr.string_whack( sp.contents )
+ return res
+
+ def WriteString( self, path, value ) :
+ rc = self.__mgr.KConfigWriteString( self.__ptr, to_char_p( path ), to_char_p( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigWriteString( '%s', '%s' )"%( path, value ), self )
+ return rc
+
+ def ReadI64( self, path ) :
+ value = c_longlong()
+ rc = self.__mgr.KConfigReadI64( self.__ptr, to_char_p( path ), byref( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigReadI64( '%s' )"%( path ), self )
+ return value.value
+
+ def ReadU64( self, path ) :
+ value = c_ulonglong()
+ rc = self.__mgr.KConfigReadU64( self.__ptr, to_char_p( path ), byref( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigReadU64( '%s' )"%( path ), self )
+ return value.value
+
+ def ReadF64( self, path ) :
+ value = c_double()
+ rc = self.__mgr.KConfigReadF64( self.__ptr, to_char_p( path ), byref( value ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigReadF64( '%s' )"%( path ), self )
+ return value.value
+
+ def DisableUserSettings( self ) :
+ rc = self.__mgr.KConfigDisableUserSettings( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigDisableUserSettings()", self )
+ return value.value
+
+ def MakeRepositoryMgr( self, mode = OpenMode.Read ) :
+ ptr = c_void_p()
+ rc = 0
+ if mode == OpenMode.Read :
+ rc = self.__mgr.KConfigMakeRepositoryMgrRead( self.__ptr, byref( ptr ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigMakeRepositoryMgrRead()", self )
+ else :
+ rc = self.__mgr.KConfigMakeRepositoryMgrUpdate( self.__ptr, byref( ptr ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "KConfigMakeRepositoryMgrUpdate()", self )
+ return KRepositoryMgr( self.__mgr, ptr )
+
+
+#------------------------------------------------------------------------------------------------------------
+class VPath :
+ def __init__( self, mgr, ptr ) :
+ self.mgr = __mgr
+ self.ptr = __ptr
+
+ def __str__( self ) :
+ if self.__ptr :
+ s = vdb_string( None, 0, 0 )
+ sp = pointer( s )
+ rc = VPathMakeString( self.__ptr, byref( sp ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VPathMakeString()", self )
+ res = sp.contents.addr
+ self.__mgr.string_whack( sp.contents )
+ else :
+ res = ""
+ return res
+
+
+#------------------------------------------------------------------------------------------------------------
+class VResolver :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.VResolverRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VResolverRelease()", self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def QueryLocal( self, path ) :
+ loc = c_void_p()
+ rc = self.__mgr.VResolverQuery( self.__ptr, c_int( 0 ), path.__ptr, byref( loc ), 0, 0 )
+ if rc != 0 :
+ return vpath( self.__mgr, 0 )
+ return vpath( self.__mgr, loc )
+
+ def QueryRemote( self, path, remote_prot = RemoteProto.Http ) :
+ rem = c_void_p()
+ rc = self.__mgr.fVResolverQuery( self.__ptr, c_int( remote_prot ), path.__ptr, 0, byref( rem ), 0 )
+ if rc != 0 :
+ return vpath( self.__mgr, 0 )
+ return vpath( self.__mgr, rem )
+
+ def QueryCache( self, path ) :
+ cache = c_void_p()
+ rc = self.__mgr.VResolverQuery( self.__ptr, c_int( 0 ), path.__ptr, 0, 0, byref( cache ) )
+ if rc != 0 :
+ return vpath( self.__mgr, 0 )
+ return vpath( self.__mgr, cache )
+
+ def RemoteEnable( self, state = ResolvEnable.UseConfig ) :
+ rc = self.__mgr.VResolverRemoteEnable( self.__ptr, c_int( state ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VResolverRemoteEnable()", self )
+
+
+#------------------------------------------------------------------------------------------------------------
+class VFSManager :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.VFSManagerRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VFSManagerRelease()", self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def GetResolver( self ) :
+ ptr = c_void_p()
+ rc = self.__mgr.VFSManagerGetResolver( self.__ptr, byref( ptr ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VFSManagerGetResolver()", self )
+ return VResolver( self.__mgr, ptr )
+
+ def MakeResolver( self, cfg ) :
+ ptr = c_void_p()
+ rc = self.__mgr.VFSManagerMakeResolver( self.__ptr, byref( ptr ), cfg.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VFSManagerMakeResolver()", self )
+ return VResolver( self.__mgr, ptr )
+
+ def MakePath( self, s_path ) :
+ path = c_void_p()
+ rc = self.__mgr.VFSManagerMakePath( self.__ptr, byref( path ), to_char_p( s_path ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "VFSManagerMakePath( '%s' )"%( s_paht ), self )
+ return VPath( self.__mgr, path )
+
+
+#------------------------------------------------------------------------------------------------------------
+class RefVariation :
+ def __init__( self, mgr, ptr ) :
+ self.__mgr = mgr
+ self.__ptr = ptr
+
+ def __del__( self ) :
+ rc = self.__mgr.RefVariationRelease( self.__ptr )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "RefVariationRelease()", self )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def GetIUPACSearchQuery( self ) :
+ txt = c_char_p()
+ len = c_int()
+ start = c_int()
+ rc = self.__mgr.RefVariationGetIUPACSearchQuery( self.__ptr, byref( txt ), byref( len ), byref( start ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "RefVariationGetIUPACSearchQuery()", self )
+ return ( txt.value, len.value, start.value )
+
+ def GetSearchQueryLenOnRef( self ) :
+ len = c_int()
+ rc = self.__mgr.RefVariationGetSearchQueryLenOnRef( self.__ptr, byref( len ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "RefVariationGetSearchQueryLenOnRef()", self )
+ return len.value
+
+ def GetAllele( self ) :
+ txt = c_char_p()
+ len = c_int()
+ start = c_int()
+ rc = self.__mgr.RefVariationGetAllele( self.__ptr, byref( txt ), byref( len ), byref( start ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "RefVariationGetAllele()", self )
+ return ( txt.value, len.value, start.value )
+
+ def GetAlleleLenOnRef( self ) :
+ len = c_int()
+ rc = self.__mgr.RefVariationGetAlleleLenOnRef( self.__ptr, byref( len ) )
+ if rc != 0 :
+ self.__mgr.raise_rc( rc, "RefVariationGetAlleleLenOnRef()", self )
+ return len.value
+
+
+#------------------------------------------------------------------------------------------------------------
+def make_lib_name( mode, prefix = None ) :
+ libname = None
+ if platform.system() == "Windows":
+ libname = "ncbi-wvdb.dll" if mode == OpenMode.Write else "ncbi-vdb.dll"
+ elif platform.system() == "Darwin":
+ libname = "libncbi-wvdb.dylib" if mode == OpenMode.Write else "libncbi-vdb.dylib"
+ elif platform.system() == "Linux" :
+ libname = "libncbi-wvdb.so" if mode == OpenMode.Write else "libncbi-vdb.so"
+ if libname == None :
+ return None
+ if prefix != None :
+ return os.path.join( os.path.sep, prefix, libname )
+ return libname
+
+def check_lib_path( mode, path ) :
+ res = None
+ if path != None :
+ if not os.path.isfile( path ) :
+ raise vdb_error( 0, "lib '%s' does not exist"% path, None )
+ else :
+ res = path
+ else :
+ res = make_lib_name( mode )
+ if res == None :
+ raise vdb_error( 0, "cannot load lib: unknow platform", None )
+ else :
+ if not os.path.isfile( res ) :
+ home_ncbi_lib64 = os.path.join( os.path.sep, os.path.expanduser( "~" ), ".ncbi", "lib64" )
+ res = make_lib_name( mode, home_ncbi_lib64 )
+ if not os.path.isfile( res ) :
+ raise vdb_error( 0, "cannot find lib: '%s'"%p, None )
+ return res
+
+
+class manager :
+ def __init__( self, mode = OpenMode.Read, path = None ) :
+ self.__mode = mode
+ self.__dir = None
+ self.__ptr = None
+
+ p = check_lib_path( mode, path )
+ try :
+ self.__lib = cdll.LoadLibrary( p )
+ except :
+ raise vdb_error( 0, "cannot load library '%s'"%p, None )
+
+ #we need this one first, because we need it to throw a vdb-error ( used in manager.explain() )
+ self.string_printf = self.__func__( "string_printf", [ c_void_p, c_int, c_void_p, c_char_p, c_int ] )
+
+ self.KDirectoryNativeDir = self.__func__( "KDirectoryNativeDir_v1", [ c_void_p ] )
+ self.KDirectoryRelease = self.__func__( "KDirectoryRelease_v1", [ c_void_p ] )
+
+ self.VDBManagerRelease = self.__func__( "VDBManagerRelease", [ c_void_p ] )
+ self.VDBManagerVersion = self.__func__( "VDBManagerVersion", [ c_void_p, c_void_p ] )
+ self.VDBManagerPathType = self.__func__( "VDBManagerPathType", [ c_void_p, c_char_p ] )
+ self.VDBManagerGetObjVersion = self.__func__( "VDBManagerGetObjVersion", [ c_void_p, c_void_p, c_char_p ] )
+ self.VDBManagerGetObjModDate = self.__func__( "VDBManagerGetObjModDate", [ c_void_p, c_void_p, c_char_p ] )
+ self.VDBManagerOpenDBRead = self.__func__( "VDBManagerOpenDBRead", [ c_void_p, c_void_p, c_void_p, c_char_p ] )
+ self.VDBManagerOpenTableRead = self.__func__( "VDBManagerOpenTableRead", [ c_void_p, c_void_p, c_void_p, c_char_p ] )
+ self.VDBManagerMakeSchema = self.__func__( "VDBManagerMakeSchema", [ c_void_p, c_void_p ] )
+
+ self.VDatabaseOpenDBRead = self.__func__( "VDatabaseOpenDBRead", [ c_void_p, c_void_p, c_char_p ] )
+ self.VDatabaseRelease = self.__func__( "VDatabaseRelease", [ c_void_p ] )
+ self.VDatabaseOpenTableRead = self.__func__( "VDatabaseOpenTableRead", [ c_void_p, c_void_p, c_char_p ] )
+ self.VDatabaseListTbl = self.__func__( "VDatabaseListTbl", [ c_void_p, c_void_p ] )
+ self.VDatabaseListDB = self.__func__( "VDatabaseListDB", [ c_void_p, c_void_p ] )
+ self.VDatabaseLocked = self.__func__( "VDatabaseLocked", [ c_void_p ], c_bool )
+
+ self.VTableRelease = self.__func__( "VTableRelease", [ c_void_p ] )
+ self.VTableListCol = self.__func__( "VTableListCol", [ c_void_p, c_void_p ] )
+ self.VTableCreateCursorRead = self.__func__( "VTableCreateCursorRead", [ c_void_p, c_void_p ] )
+ self.VTableCreateCachedCursorRead = self.__func__( "VTableCreateCachedCursorRead", [ c_void_p, c_void_p, c_longlong ] )
+ self.VTableLocked = self.__func__( "VTableLocked", [ c_void_p ], c_bool )
+ self.VTableOpenKTableRead = self.__func__( "VTableOpenKTableRead", [ c_void_p, c_void_p ] )
+
+ self.KNamelistCount = self.__func__( "KNamelistCount", [ c_void_p, c_void_p ] )
+ self.KNamelistGet = self.__func__( "KNamelistGet", [ c_void_p, c_int, c_void_p ] )
+ self.KNamelistRelease = self.__func__( "KNamelistRelease", [ c_void_p ] )
+
+ self.VSchemaRelease = self.__func__( "VSchemaRelease", [ c_void_p ] )
+ self.VSchemaParseText = self.__func__( "VSchemaParseText", [ c_void_p, c_char_p, c_char_p, c_int ] )
+ self.VSchemaParseFile = self.__func__( "VSchemaParseFile", [ c_void_p, c_char_p ] )
+ self.VSchemaAddIncludePath = self.__func__( "VSchemaAddIncludePath", [ c_void_p, c_char_p ] )
+
+ self.VCursorRelease = self.__func__( "VCursorRelease", [ c_void_p ] )
+ self.VCursorAddColumn = self.__func__( "VCursorAddColumn", [ c_void_p, c_void_p, c_char_p ] )
+ self.VCursorOpen = self.__func__( "VCursorOpen", [ c_void_p ] )
+ self.VCursorDatatype = self.__func__( "VCursorDatatype", [ c_void_p, c_int, c_void_p, c_void_p ] )
+ self.VCursorCellDataDirect = self.__func__( "VCursorCellDataDirect", [ c_void_p, c_longlong, c_int, c_void_p, c_void_p, c_void_p, c_void_p ] )
+ self.VCursorCloseRow = self.__func__( "VCursorCloseRow", [ c_void_p ] )
+ self.VCursorIdRange = self.__func__( "VCursorIdRange", [ c_void_p, c_int, c_void_p, c_void_p ] )
+ self.VCursorRowId = self.__func__( "VCursorRowId", [ c_void_p, c_void_p ] )
+ self.VCursorOpenRow = self.__func__( "VCursorOpenRow", [ c_void_p ] )
+ self.VCursorFindNextRowIdDirect = self.__func__( "VCursorFindNextRowIdDirect", [ c_void_p, c_int, c_longlong, c_void_p ] )
+
+ self.KTableRelease = self.__func__( "KTableRelease", [ c_void_p ] )
+ self.KTableListIdx = self.__func__( "KTableListIdx", [ c_void_p, c_void_p ] )
+ self.KTableOpenIndexRead = self.__func__( "KTableOpenIndexRead", [ c_void_p, c_void_p, c_char_p ] )
+ self.KTableOpenMetadataRead = self.__func__( "KTableOpenMetadataRead", [ c_void_p, c_void_p ] )
+
+ self.KIndexRelease = self.__func__( "KIndexRelease", [ c_void_p ] )
+ self.KIndexVersion = self.__func__( "KIndexVersion", [ c_void_p, c_void_p ] )
+ self.KIndexType = self.__func__( "KIndexType", [ c_void_p, c_void_p ] )
+ self.KIndexLocked = self.__func__( "KIndexLocked", [ c_void_p ], c_bool )
+ self.KIndexFindText = self.__func__( "KIndexFindText", [ c_void_p, c_char_p, c_void_p, c_void_p, c_void_p, c_void_p ] )
+ self.KIndexProjectText = self.__func__( "KIndexProjectText", [ c_void_p, c_longlong, c_void_p, c_void_p, c_char_p, c_int, c_void_p ] )
+
+ self.KMetadataRelease = self.__func__( "KMetadataRelease", [ c_void_p ] )
+ self.KMetadataVersion = self.__func__( "KMetadataVersion", [ c_void_p, c_void_p ] )
+ self.KMetadataByteOrder = self.__func__( "KMetadataByteOrder", [ c_void_p, c_void_p ] )
+ self.KMetadataRevision = self.__func__( "KMetadataRevision", [ c_void_p, c_void_p ] )
+ self.KMetadataMaxRevision = self.__func__( "KMetadataMaxRevision", [ c_void_p, c_void_p ] )
+ self.KMetadataOpenRevision = self.__func__( "KMetadataOpenRevision", [ c_void_p, c_void_p, c_int ] )
+ self.KMetadataGetSequence = self.__func__( "KMetadataGetSequence", [ c_void_p, c_char_p, c_void_p ] )
+ self.KMetadataOpenNodeRead = self.__func__( "KMetadataOpenNodeRead", [ c_void_p, c_void_p, c_char_p ] )
+
+ self.KMDataNodeOpenNodeRead = self.__func__( "KMDataNodeOpenNodeRead", [ c_void_p, c_void_p, c_char_p ] )
+ self.KMDataNodeRelease = self.__func__( "KMDataNodeRelease", [ c_void_p ] )
+ self.KMDataNodeListAttr = self.__func__( "KMDataNodeListAttr", [ c_void_p ] )
+ self.KMDataNodeListChildren = self.__func__( "KMDataNodeListChildren", [ c_void_p ] )
+ self.KMDataNodeByteOrder = self.__func__( "KMDataNodeByteOrder", [ c_void_p, c_void_p ] )
+ self.KMDataNodeRead = self.__func__( "KMDataNodeRead", [ c_void_p, c_size_t, c_void_p, c_size_t, c_void_p, c_void_p ] )
+
+ if mode == OpenMode.Write :
+ self.VDBManagerMakeUpdate = self.__func__( "VDBManagerMakeUpdate", [ c_void_p, c_void_p ] )
+ self.VDBManagerCreateTable = self.__func__( "VDBManagerCreateTable", [ c_void_p, c_void_p, c_void_p, c_char_p, c_int, c_char_p ] )
+ self.VDBManagerCreateDB = self.__func__( "VDBManagerCreateDB", [ c_void_p, c_void_p, c_void_p, c_char_p, c_int, c_char_p ] )
+ self.VDBManagerOpenTableUpdate = self.__func__( "VDBManagerOpenTableUpdate", [ c_void_p, c_void_p, c_void_p, c_char_p ] )
+ self.VDBManagerOpenDBUpdate = self.__func__( "VDBManagerOpenDBUpdate", [ c_void_p, c_void_p, c_void_p, c_char_p ] )
+
+ self.VTableCreateCursorWrite = self.__func__( "VTableCreateCursorWrite", [ c_void_p, c_void_p, c_int ] )
+ self.VTableOpenKTableUpdate = self.__func__( "VTableOpenKTableUpdate", [ c_void_p, c_void_p ] )
+
+ self.VCursorCommitRow = self.__func__( "VCursorCommitRow", [ c_void_p ] )
+ self.VCursorRepeatRow = self.__func__( "VCursorRepeatRow", [ c_void_p, c_longlong ] )
+ self.VCursorFlushPage = self.__func__( "VCursorFlushPage", [ c_void_p ] )
+ self.VCursorCommit = self.__func__( "VCursorCommit", [ c_void_p ] )
+ self.VCursorWrite = self.__func__( "VCursorWrite", [ c_void_p, c_int, c_int, c_void_p, c_int, c_int ] )
+ self.VCursorDefault = self.__func__( "VCursorDefault", [ c_void_p, c_int, c_int, c_void_p, c_int, c_int ] )
+
+ self.VDatabaseCreateDB = self.__func__( "VDatabaseCreateDB", [ c_void_p, c_void_p, c_char_p, c_int, c_char_p ] )
+ self.VDatabaseOpenTableUpdate = self.__func__( "VDatabaseOpenTableUpdate", [ c_void_p, c_void_p, c_char_p ] )
+ self.VDatabaseOpenDBUpdate = self.__func__( "VDatabaseOpenDBUpdate", [ c_void_p, c_void_p, c_char_p ] )
+ self.VDatabaseCreateTable = self.__func__( "VDatabaseCreateTable", [ c_void_p, c_void_p, c_char_p, c_int, c_char_p ] )
+ self.VDatabaseDropDB = self.__func__( "VDatabaseDropDB", [ c_void_p, c_char_p ] )
+ self.VDatabaseDropTable = self.__func__( "VDatabaseDropTable", [ c_void_p, c_char_p ] )
+ self.VDatabaseLock = self.__func__( "VDatabaseLock", [ c_void_p, c_int, c_char_p ] )
+ self.VDatabaseUnlock = self.__func__( "VDatabaseUnlock", [ c_void_p, c_int, c_char_p ] )
+
+ self.KTableCreateIndex = self.__func__( "KTableCreateIndex", [ c_void_p, c_void_p, c_int, c_int, c_char_p ] )
+ self.KIndexCommit = self.__func__( "KIndexCommit", [ c_void_p ] )
+ self.KIndexInsertText = self.__func__( "KIndexInsertText", [ c_void_p, c_bool, c_char_p, c_longlong ] )
+ self.KIndexDeleteText = self.__func__( "KIndexDeleteText", [ c_void_p, c_char_p ] )
+ self.KIndexInsertU64 = self.__func__( "KIndexInsertU64", [ c_void_p, c_bool, c_longlong, c_longlong, c_longlong, c_longlong ] )
+ self.KIndexDeleteU64 = self.__func__( "KIndexDeleteU64", [ c_void_p, c_longlong ] )
+
+ self.KTableOpenMetadataUpdate = self.__func__( "KTableOpenMetadataUpdate", [ c_void_p, c_void_p ] )
+
+ self.KMetadataCommit = self.__func__( "KMetadataCommit", [ c_void_p ] )
+ self.KMetadataFreeze = self.__func__( "KMetadataFreeze", [ c_void_p ] )
+ self.KMetadataSetSequence = self.__func__( "KMetadataSetSequence", [ c_void_p, c_char_p, c_longlong ] )
+ self.KMetadataNextSequence = self.__func__( "KMetadataNextSequence", [ c_void_p, c_char_p, c_void_p ] )
+ self.KMetadataOpenNodeUpdate = self.__func__( "KMetadataOpenNodeUpdate", [ c_void_p, c_void_p, c_char_p ] )
+
+ self.KMDataNodeOpenNodeUpdate = self.__func__( "KMDataNodeOpenNodeUpdate", [ c_void_p, c_void_p, c_char_p ] )
+ self.KMDataNodeWrite = self.__func__( "KMDataNodeWrite", [ c_void_p, c_void_p, c_size_t ] )
+ self.KMDataNodeAppend = self.__func__( "KMDataNodeAppend", [ c_void_p, c_void_p, c_size_t ] )
+
+ else :
+ self.VDBManagerMakeRead = self.__func__( "VDBManagerMakeRead", [ c_void_p, c_void_p ] )
+
+ self.KConfigMake = self.__func__( "KConfigMake", [ c_void_p, c_void_p ] )
+ self.KConfigRelease = self.__func__( "KConfigRelease", [ c_void_p ] )
+ self.KConfigCommit = self.__func__( "KConfigCommit", [ c_void_p ] )
+ self.KConfigReadBool = self.__func__( "KConfigReadBool", [ c_void_p, c_char_p, c_void_p ] )
+ self.KConfigWriteBool = self.__func__( "KConfigWriteBool", [ c_void_p, c_char_p, c_bool ] )
+ self.KConfigReadString = self.__func__( "KConfigReadString", [ c_void_p, c_char_p, c_void_p ] )
+ self.KConfigWriteString = self.__func__( "KConfigWriteString", [ c_void_p, c_char_p, c_char_p ] )
+ self.KConfigReadI64 = self.__func__( "KConfigReadI64", [ c_void_p, c_char_p, c_void_p ] )
+ self.KConfigReadU64 = self.__func__( "KConfigReadU64", [ c_void_p, c_char_p, c_void_p ] )
+ self.KConfigReadF64 = self.__func__( "KConfigReadF64", [ c_void_p, c_char_p, c_void_p ] )
+ self.KConfigDisableUserSettings = self.__func__( "KConfigDisableUserSettings", [] )
+ self.KConfigMakeRepositoryMgrRead = self.__func__( "KConfigMakeRepositoryMgrRead", [ c_void_p, c_void_p ] )
+ self.KConfigMakeRepositoryMgrUpdate = self.__func__( "KConfigMakeRepositoryMgrUpdate", [ c_void_p, c_void_p ] )
+
+ self.KRepositoryMgrRelease = self.__func__( "KRepositoryMgrRelease", [ c_void_p ] )
+ self.KRepositoryMgrHasRemoteAccess = self.__func__( "KRepositoryMgrHasRemoteAccess", [ c_void_p ], c_bool )
+ self.KRepositoryMgrUserRepositories = self.__func__( "KRepositoryMgrUserRepositories", [ c_void_p, c_void_p ] )
+ self.KRepositoryMgrSiteRepositories = self.__func__( "KRepositoryMgrSiteRepositories", [ c_void_p, c_void_p ] )
+ self.KRepositoryMgrRemoteRepositories = self.__func__( "KRepositoryMgrRemoteRepositories", [ c_void_p, c_void_p ] )
+ self.KRepositoryMgrCategoryDisabled = self.__func__( "KRepositoryMgrCategoryDisabled", [ c_void_p, c_int ], c_bool )
+ self.KRepositoryMgrCategorySetDisabled = self.__func__( "KRepositoryMgrCategorySetDisabled", [ c_void_p, c_int, c_bool ] )
+ self.KRepositoryVectorWhack = self.__func__( "KRepositoryVectorWhack", [ c_void_p ] )
+ self.KRepositoryAddRef = self.__func__( "KRepositoryAddRef", [ c_void_p ] )
+ self.KRepositoryRelease = self.__func__( "KRepositoryRelease", [ c_void_p ] )
+ self.KRepositoryCategory = self.__func__( "KRepositoryCategory", [ c_void_p ] )
+ self.KRepositorySubCategory = self.__func__( "KRepositorySubCategory", [ c_void_p ] )
+ self.KRepositoryName = self.__func__( "KRepositoryName", [ c_void_p, c_char_p, c_int, c_void_p ] )
+ self.KRepositoryDisplayName = self.__func__( "KRepositoryDisplayName", [ c_void_p, c_char_p, c_int, c_void_p ] )
+ self.KRepositoryRoot = self.__func__( "KRepositoryRoot", [ c_void_p, c_char_p, c_int, c_void_p ] )
+ self.KRepositorySetRoot = self.__func__( "KRepositorySetRoot", [ c_void_p, c_char_p, c_int ] )
+ self.KRepositoryResolver = self.__func__( "KRepositoryResolver", [ c_void_p, c_char_p, c_int, c_void_p ] )
+ self.KRepositoryDisabled = self.__func__( "KRepositoryDisabled", [ c_void_p ], c_bool )
+ self.KRepositorySetDisabled = self.__func__( "KRepositorySetDisabled", [ c_void_p, c_bool ] )
+ self.KRepositoryCacheEnabled = self.__func__( "KRepositoryCacheEnabled", [ c_void_p ], c_bool )
+
+ self.VFSManagerMake = self.__func__( "VFSManagerMake", [ c_void_p ] )
+ self.VFSManagerRelease = self.__func__( "VFSManagerRelease", [ c_void_p ] )
+ self.VFSManagerGetResolver = self.__func__( "VFSManagerGetResolver", [ c_void_p, c_void_p ] )
+ self.VFSManagerMakeResolver = self.__func__( "VFSManagerMakeResolver", [ c_void_p, c_void_p, c_void_p ] )
+ self.VFSManagerMakePath = self.__func__( "VFSManagerMakePath", [ c_void_p, c_void_p, c_void_p ] )
+
+ self.VResolverRelease = self.__func__( "VResolverRelease", [ c_void_p ] )
+ self.VResolverQuery = self.__func__( "VResolverQuery", [ c_void_p, c_int, c_void_p, c_void_p, c_void_p, c_void_p ] )
+ self.VResolverRemoteEnable = self.__func__( "VResolverRemoteEnable", [ c_void_p, c_int ] )
+
+ self.RefVariationIUPACMake = self.__func__( "RefVariationIUPACMake", [ c_void_p, c_char_p, c_longlong, c_longlong, c_longlong, c_char_p, c_longlong, c_int ] )
+ self.RefVariationRelease = self.__func__( "RefVariationRelease", [ c_void_p ] )
+ self.RefVariationGetIUPACSearchQuery = self.__func__( "RefVariationGetIUPACSearchQuery", [ c_void_p, c_void_p, c_void_p, c_void_p ] )
+ self.RefVariationGetSearchQueryLenOnRef = self.__func__( "RefVariationGetSearchQueryLenOnRef", [ c_void_p, c_void_p ] )
+ self.RefVariationGetAllele = self.__func__( "RefVariationGetAllele", [ c_void_p, c_void_p, c_void_p, c_void_p ] )
+ self.RefVariationGetAlleleLenOnRef = self.__func__( "RefVariationGetAlleleLenOnRef", [ c_void_p, c_void_p ] )
+
+ self.ReferenceList_MakeCursor = self.__func__( "ReferenceList_MakeCursor", [ c_void_p, c_void_p, c_int, c_char_p, c_int ] )
+ self.ReferenceList_MakeTable = self.__func__( "ReferenceList_MakeTable", [ c_void_p, c_void_p, c_int, c_int, c_char_p, c_int ] )
+ self.ReferenceList_MakeDatabase = self.__func__( "ReferenceList_MakeDatabase", [ c_void_p, c_void_p, c_int, c_int, c_char_p, c_int ] )
+ self.ReferenceList_MakePath = self.__func__( "ReferenceList_MakePath", [ c_void_p, c_void_p, c_char_p, c_int, c_int, c_char_p, c_int ] )
+ self.ReferenceList_Release = self.__func__( "ReferenceList_Release", [ c_void_p ] )
+ self.ReferenceList_Count = self.__func__( "ReferenceList_Count", [ c_void_p, c_void_p ] )
+ self.ReferenceList_Find = self.__func__( "ReferenceList_Find", [ c_void_p, c_void_p, c_char_p, c_int ] )
+ self.ReferenceList_Get = self.__func__( "ReferenceList_Get", [ c_void_p, c_void_p, c_int ] )
+
+ self.ReferenceObj_Release = self.__func__( "ReferenceObj_Release", [ c_void_p ] )
+ self.ReferenceObj_Idx = self.__func__( "ReferenceObj_Idx", [ c_void_p, c_void_p ] )
+ self.ReferenceObj_IdRange = self.__func__( "ReferenceObj_IdRange", [ c_void_p, c_void_p, c_void_p ] )
+ self.ReferenceObj_Bin = self.__func__( "ReferenceObj_Bin", [ c_void_p, c_void_p ] )
+ self.ReferenceObj_SeqId = self.__func__( "ReferenceObj_SeqId", [ c_void_p, c_void_p ] )
+ self.ReferenceObj_Name = self.__func__( "ReferenceObj_Name", [ c_void_p, c_void_p ] )
+ self.ReferenceObj_SeqLength = self.__func__( "ReferenceObj_SeqLength", [ c_void_p, c_void_p ] )
+ self.ReferenceObj_Circular = self.__func__( "ReferenceObj_Circular", [ c_void_p, c_void_p ] )
+ self.ReferenceObj_External = self.__func__( "ReferenceObj_External", [ c_void_p, c_void_p, c_void_p ] )
+ self.ReferenceObj_Read = self.__func__( "ReferenceObj_Read", [ c_void_p, c_int, c_int, c_void_p, c_void_p ] )
+ self.ReferenceObj_GetIdCount = self.__func__( "ReferenceObj_GetIdCount", [ c_void_p, c_longlong, c_void_p ] )
+
+ self.VPathMakeString = self.__func__( "VPathMakeString", [ c_void_p, c_void_p ] )
+ self.VectorGet = self.__func__( "VectorGet", [ c_void_p, c_int ], c_void_p )
+ self.StringWhack = self.__func__( "StringWhack", [ c_void_p ] )
+
+ self.__dir = self.__make_native_dir__()
+ self.__ptr = self.__make_mgr__( mode )
+
+
+ def __del__( self ) :
+ if self.__ptr != None :
+ if self.VDBManagerRelease != None :
+ self.VDBManagerRelease( self.__ptr )
+ if self.__dir != None :
+ if self.KDirectoryRelease != None :
+ self.KDirectoryRelease( self.__dir )
+
+ def __enter__( self ) :
+ return self
+
+ def __exit__( self, type, value, traceback ) :
+ pass
+
+ def string_whack( self, str ) :
+ self.StringWhack( byref( str ) )
+
+ def __make_native_dir__( self ) :
+ res = c_void_p()
+ if self.KDirectoryNativeDir == None :
+ raise "'KDirectoryNativeDir' not found in lib"
+ else :
+ rc = self.KDirectoryNativeDir( byref( res ) )
+ if rc != 0 :
+ self.raise_rc( rc, "KDirectoryNativeDir()", self )
+ return res;
+
+ def __make_mgr__( self, mode ) :
+ res = c_void_p()
+ if mode == OpenMode.Write :
+ rc = self.VDBManagerMakeUpdate( byref( res ), self.__dir )
+ else :
+ rc = self.VDBManagerMakeRead( byref( res ), self.__dir )
+ if rc != 0 :
+ self.raise_rc( rc, "make_mgr()", self )
+ return res
+
+ def __func__( self, name, argt, rest = c_int ) :
+ res = None
+ try :
+ res = getattr( self.__lib, name )
+ res.argtypes = argt
+ res.restype = rest
+ except :
+ pass
+ return res
+
+ def explain( self, rc ) :
+ buffer = create_string_buffer( 1024 )
+ num_writ = c_int( 0 )
+ fmt = create_string_buffer( to_bytes( "%R" ) )
+ if self.string_printf( buffer, c_int( 1024 ), byref( num_writ ), fmt, c_int( rc ) ) == 0 :
+ if PY3 :
+ return buffer.value.decode( "utf-8" )
+ return buffer.value
+ return "cannot explain %d"%( rc )
+
+ def raise_rc( self, rc, funcname, obj ) :
+ msg = "%s -> %s"%( funcname, self.explain( rc ) )
+ raise vdb_error( rc, msg, obj )
+
+ def Version( self ) :
+ vers = c_int( 0 )
+ rc = self.VDBManagerVersion( self.__ptr, byref( vers ) )
+ if rc != 0 :
+ self.raise_rc( rc, "version()", self )
+ return version( vers.value )
+
+ def writable( self ) :
+ return self.__mode == OpenMode.Write
+
+ def PathType( self, path ) :
+ return PathType( self.VDBManagerPathType( self.__ptr, to_char_p( path ) ) )
+
+ def GetObjVersion( self, path ) :
+ vers = c_int()
+ rc = self.VDBManagerGetObjVersion( self.__ptr, byref( vers ), to_char_p( path ) )
+ if rc != 0 :
+ self.raise_rc( rc, "obj_vers( '%s' )"%( path ), self )
+ return version( vers.value )
+
+ def GetObjModDate( self, path ) :
+ t = c_int()
+ rc = self.VDBManagerGetObjModDate( self.__ptr, byref( t ), to_char_p( path ) )
+ if rc != 0 :
+ self.raise_rc( rc, "obj_time( '%s' )"%( path ), self )
+ return datetime.datetime.fromtimestamp( t.value )
+
+ def OpenDB( self, path, writable = False, schema = None ) :
+ f = self.VDBManagerOpenDBUpdate if writable else self.VDBManagerOpenDBRead
+ db = c_void_p()
+ vdb_schema_ptr = schema.__ptr if schema != None else c_void_p( 0 )
+ rc = f( self.__ptr, byref( db ), vdb_schema_ptr, to_char_p( path ) )
+ if rc != 0 :
+ self.raise_rc( rc, "open_db( %s )"%( path ), self )
+ return VDatabase( self, db, path )
+
+ def OpenTable( self, path, writable = False, schema = None ) :
+ f = self.VDBManagerOpenTableUpdate if writable else self.VDBManagerOpenTableRead
+ tab = c_void_p()
+ vdb_schema_ptr = schema.__ptr if schema != None else c_void_p( 0 )
+ rc = f( self.__ptr, byref( tab ), vdb_schema_ptr, to_char_p( path ) )
+ if rc != 0 :
+ self.raise_rc( rc, "VDBManagerOpenTable( %s )"%( path ), self )
+ return VTable( self, tab, path )
+
+ def CreateDB( self, schema, spec, path = None, mode = CreateMode.Init ) :
+ if schema == None :
+ raise vdb_error( "CreateDB(): schema missing", self, 0 )
+ if spec == None :
+ raise vdb_error( "CreateDB(): spec missing", self, 0 )
+ db = c_void_p()
+ px = path if path != None else spec
+ p = to_char_p( px )
+ rc = self.VDBManagerCreateDB( self.__ptr, byref( db ), schema._VSchema__ptr, to_char_p( spec ), c_int( mode.value ), p )
+ if rc != 0 :
+ self.raise_rc( rc, "vdb_manger.VDBManagerCreateDB( %s )"%( px ), self )
+ return VDatabase( self, db, px )
+
+ def CreateTable( self, schema, spec, path = None, mode = CreateMode.Init ) :
+ if schema == None :
+ raise vdb_error( "CreateTable(): schema missing", self, 0 )
+ if spec == None :
+ raise vdb_error( "CreateTable(): spec missing", self, 0 )
+ tab = c_void_p()
+ px = path if path != None else spec
+ p = to_char_p( px )
+ rc = self.VDBManagerCreateTable( self.__ptr, byref( tab ), schema._VSchema__ptr, to_char_p( spec ), c_int( mode.value ), p )
+ if rc != 0 :
+ self.raise_rc( rc, "VDBManagerCreateTable( %s )"%( px ), self )
+ return VTable( self, tab, px )
+
+ def MakeSchema( self, schema_text = None ) :
+ vdb_schema_ptr = c_void_p()
+ rc = self.VDBManagerMakeSchema( self.__ptr, byref( vdb_schema_ptr ) )
+ if rc != 0 :
+ self.raise_rc( rc, "VDBManagerMakeSchema()", self )
+ res = VSchema( self, vdb_schema_ptr )
+ if schema_text != None :
+ res.ParseText( schema_text )
+ return res
+
+ def MakeKConfig( self ) :
+ ptr = c_void_p()
+ rc = self.KConfigMake( byref( ptr ), c_void_p( 0 ) )
+ if rc != 0 :
+ self.raise_rc( rc, "KConfigMake()", self )
+ return KConfig( self, ptr )
+
+ def MakeVFSManager( self ) :
+ ptr = c_void_p()
+ rc = self.VFSManagerMake( byref( ptr ) )
+ if rc != 0 :
+ self.raise_rc( rc, "VFSManagerMake()", self )
+ return VFSManager( self, ptr )
+
+ def ReferenceList( self, path, options = 2 ) :
+ ptr = c_void_p()
+ rc = self.ReferenceList_MakePath( byref( ptr ), self.__ptr, to_char_p( path ), c_int( options ), c_int( 0 ), c_char_p( 0 ), c_int( 0 ) )
+ if rc != 0 :
+ self.raise_rc( rc, "ReferenceList_MakePath( '%s' )" % path, self )
+ return ReferenceList( self, ptr )
+
+ def RefVariation( self, ref_bases, del_pos, del_len, insertion, algo = 2 ) :
+ ptr = c_void_p()
+ ref = to_char_p( ref_bases )
+ ref_len = c_longlong( len( ref_bases ) )
+ ins = to_char_p( insertion )
+ ins_len = c_longlong( len( insertion ) )
+ rc = self.RefVariationIUPACMake( byref( ptr ), ref, ref_len, c_longlong( del_pos ), c_longlong( del_len ), ins, ins_len, c_int( algo ) )
+ if rc != 0 :
+ self.raise_rc( rc, "RefVariationIUPACMake( %s )"%( ref_bases ), self )
+ return RefVariation( self, ptr )
diff --git a/setup/konfigure.perl b/setup/konfigure.perl
index 5568cdd..532e616 100644
--- a/setup/konfigure.perl
+++ b/setup/konfigure.perl
@@ -438,7 +438,8 @@ my @dependencies;
my %DEPEND_OPTIONS;
foreach my $href (DEPENDS()) {
$_ = $href->{name};
- my ($I, $L) = ($href->{Include});
+ my $I = $href->{Include};
+ my @L;
my $o = "with-$_-prefix";
++$DEPEND_OPTIONS{$o};
if ($OPT{$o}) {
@@ -448,9 +449,10 @@ foreach my $href (DEPENDS()) {
my $t = File::Spec->catdir($I, 'libxml2');
$I = $t if (-e $t);
}
- $L = File::Spec->catdir($OPT{$o}, 'lib');
+ push ( @L, File::Spec->catdir($OPT{$o}, 'lib') );
+ push ( @L, File::Spec->catdir($OPT{$o}, 'lib64') );
}
- my ($i, $l) = find_lib($_, $I, $L);
+ my ($i, $l) = find_lib($_, $I, @L);
if (defined $i || $l) {
my $d = 'HAVE_' . uc($_) . ' = 1';
push @dependencies, $d;
@@ -485,6 +487,7 @@ foreach my $href (@REQ) {
my $need_build = $a{type} =~ /B/;
my $need_lib = $a{type} =~ /L|D/;
my $need_itf = ! ($a{type} =~ /D/ || $a{type} =~ /E/ || $a{type} =~ /J/);
+ $need_itf = 1 if ($a{type} =~ /I/);
my $need_jar = $a{type} =~ /J/;
my ($bin, $inc, $lib, $ilib, $src)
@@ -653,6 +656,9 @@ foreach my $href (@REQ) {
} elsif ($quasi_optional && $found_itf && ($need_lib && ! $found_lib)) {
println "configure: $a{name} package: "
. "found interface files but not libraries.";
+ $found_itf = abs_path($found_itf);
+ push(@dependencies, "$a{aname}_INCDIR = $found_itf");
+ println "includes: $found_itf";
} else {
if ($OPT{'debug'}) {
$_ = "$a{name}: includes: ";
@@ -1442,17 +1448,33 @@ sub check_tool {
}
}
-sub check_no_array_bounds {
- check_compiler('O', '-Wno-array-bounds');
-}
-
sub check_static_libstdcpp {
my $option = '-static-libstdc++';
- my $save = $TOOLS;
- $TOOLS = $CPP;
- $_ = check_compiler('O', $option);
- $TOOLS = $save;
- $_ ? $option : ''
+
+ print "checking whether $CPP accepts $option... ";
+
+ my $log = 'int main() {}\n';
+ my $cmd = $log;
+ $cmd =~ s/\\n/\n/g;
+ my $gcc = "echo -e '$log' | $CPP -xc $option - 2>&1";
+ print "\n\t\trunning $gcc\n" if ($OPT{'debug'});
+ my $out = `$gcc`;
+ my $ok = $? == 0;
+ if ( $ok && $out ) {
+ $ok = 0 if ( $out =~ /unrecognized option '-static-libstdc\+\+'/ );
+ }
+ print "$out\t" if ($OPT{'debug'});
+ println $ok ? 'yes' : 'no';
+
+ unlink 'a.out';
+
+ return '' if (!$ok);
+
+ return $option;
+}
+
+sub check_no_array_bounds {
+ check_compiler('O', '-Wno-array-bounds');
}
sub find_lib {
@@ -1460,7 +1482,7 @@ sub find_lib {
}
sub check_compiler {
- my ($t, $n, $i, $l) = @_;
+ my ($t, $n, $I, @l) = @_;
my $tool = $TOOLS;
if ($t eq 'L') {
@@ -1506,37 +1528,54 @@ sub check_compiler {
return;
}
- if ($i && ! -d $i) {
- print "'$i': " if ($OPT{'debug'});
+ if ($I && ! -d $I) {
+ print "'$I': " if ($OPT{'debug'});
println 'no';
return;
}
- if ($l && ! -d $l) {
- print "'$l': " if ($OPT{'debug'}); println 'no';
- return;
+
+ for ( my $i = 0; $i <= $#l; ++ $i ) {
+ print "'$l[$i]': " if ($OPT{'debug'});
+ if ( $l [ $i ] ) {
+ if ( -d $l [ $i ] ) {
+ last;
+ } elsif ( $i == $#l ) {
+ println 'no';
+ return;
+ }
+ }
}
my $cmd = $log;
$cmd =~ s/\\n/\n/g;
- my $gcc = "| $tool -xc $flags " . ($i ? "-I$i " : ' ')
+ push ( @l, '' ) unless ( @l );
+ for my $i ( 0 .. $#l ) {
+ my $l = $l [ $i ];
+ next if ( $l && ! -d $l );
+ my $gcc = "| $tool -xc $flags " . ($I ? "-I$I " : ' ')
. ($l ? "-L$l " : ' ') . "- $library";
- $gcc .= ' 2> /dev/null' unless ($OPT{'debug'});
+ $gcc .= ' 2> /dev/null' unless ($OPT{'debug'});
- open GCC, $gcc or last;
- print "\n\t\trunning echo -e '$log' $gcc\n" if ($OPT{'debug'});
- print GCC "$cmd" or last;
- my $ok = close GCC;
- print "\t" if ($OPT{'debug'});
- println $ok ? 'yes' : 'no';
+ open GCC, $gcc or last;
+ print "\n\t\trunning echo -e '$log' $gcc\n" if ($OPT{'debug'});
+ print GCC "$cmd" or last;
+ my $ok = close GCC;
+ print "\t" if ($OPT{'debug'});
+ if ( $ok ) {
+ println 'yes';
+ } else {
+ println 'no' if ( $i == $#l );
+ }
- unlink 'a.out';
+ unlink 'a.out';
- return if (!$ok);
+ return if ( ! $ok && ( $i == $#l ) );
- return 1 if ($t eq 'O');
+ return 1 if ($t eq 'O');
- return ($i, $l);
+ return ($I, $l) if ( $ok) ;
+ }
}
println "cannot run $tool: skipped";
diff --git a/setup/package.prl b/setup/package.prl
index fa4d60e..69f6b97 100644
--- a/setup/package.prl
+++ b/setup/package.prl
@@ -9,7 +9,8 @@ sub PKG { ( LNG => 'C',
LOCOUT=> '../../OUTDIR',
PATH => '/usr/local/ncbi/ncbi-vdb',
UPATH => '$HOME/ncbi/ncbi-vdb', ) }
-sub DEPENDS { ( { name => 'hdf5' , Include => '/usr/include' , },
+sub DEPENDS { ( { name => 'fuse' , Include => '/usr/include' , },
+ { name => 'hdf5' , Include => '/usr/include' , },
{ name => 'magic', Include => '/usr/include' , },
{ name => 'xml2' , Include => '/usr/include/libxml2', } ) }
sub REQ { ( { name => 'ngs-sdk',
@@ -45,6 +46,15 @@ sub REQ { ( { name => 'ngs-sdk',
include => 'hdf5.h',
lib => 'libhdf5.a',
},
+ { name => 'magic',
+ option => 'with-magic-prefix',
+ origin => 'E',
+ type => 'LIO',
+ pkgpath => '/usr',
+ usrpath => '$HOME',
+ include => 'magic.h',
+ lib => 'libmagic.so',
+ },
{ name => 'xml2',
option => 'with-xml2-prefix',
origin => 'E',
diff --git a/test/Makefile b/test/Makefile
index dd3ab34..955f8f4 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -59,6 +59,7 @@ SUBDIRS = \
loader \
krypto \
cipher \
+ install \
# common targets for non-leaf Makefiles; must follow a definition of SUBDIRS
include $(TOP)/build/Makefile.targets
diff --git a/build/Makefile.vers b/test/align-access/Makefile
similarity index 68%
copy from build/Makefile.vers
copy to test/align-access/Makefile
index 499fbe8..e8ba292 100644
--- a/build/Makefile.vers
+++ b/test/align-access/Makefile
@@ -22,5 +22,36 @@
#
# ===========================================================================
-# NCBI-VDB and library version
-VERSION = 2.8.1
+
+default: runtests
+
+TOP ?= $(abspath ../..)
+MODULE = test/align-access
+
+TEST_TOOLS = \
+ test-align-access \
+
+include $(TOP)/build/Makefile.env
+
+$(TEST_TOOLS): makedirs
+ @ $(MAKE_CMD) $(TEST_BINDIR)/$@
+
+clean: stdclean
+#-------------------------------------------------------------------------------
+# test-load-index
+#
+TEST_INDEX_SRC = \
+ test \
+ main
+
+TEST_INDEX_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_INDEX_SRC))
+
+TEST_INDEX_LIB = \
+ -skapp \
+ -sktst \
+ -sncbi-vdb
+
+$(TEST_BINDIR)/test-align-access: $(TEST_INDEX_OBJ)
+ $(LP) --exe -o $@ $^ $(TEST_INDEX_LIB)
+
diff --git a/test/align-access/align-access.hpp b/test/align-access/align-access.hpp
new file mode 100644
index 0000000..5181951
--- /dev/null
+++ b/test/align-access/align-access.hpp
@@ -0,0 +1,182 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+extern "C" {
+#include <klib/rc.h>
+#include <align/bam.h>
+#include <align/align-access.h>
+#include <vfs/path.h>
+#include <vfs/manager.h>
+
+ rc_t AlignAccessAlignmentEnumeratorGetSAM(const AlignAccessAlignmentEnumerator *self, size_t *const actSize, size_t const maxsize, char *const buffer);
+}
+
+namespace AlignAccess {
+ class AlignmentEnumerator;
+ class Database;
+ class Manager;
+
+ class AlignmentEnumerator {
+ friend class Database;
+ AlignAccessAlignmentEnumerator *const self;
+ mutable bool skip;
+ explicit AlignmentEnumerator(AlignAccessAlignmentEnumerator *Self) : self(Self), skip(true) {}
+ public:
+ AlignmentEnumerator(AlignmentEnumerator const &rhs) : self(rhs.self) {
+ rc_t const rc = AlignAccessAlignmentEnumeratorAddRef(self);
+ if (rc) throw std::logic_error("AlignAccessAlignmentEnumeratorAddRef failed");
+ }
+ ~AlignmentEnumerator() {
+ AlignAccessAlignmentEnumeratorRelease(self);
+ }
+ bool next() const {
+ if (self) {
+ if (skip) {
+ skip = false;
+ return true;
+ }
+ rc_t const rc = AlignAccessAlignmentEnumeratorNext(self);
+ if (rc == 0) return true;
+ if ((int)GetRCObject(rc) == rcRow && (int)GetRCState(rc) == rcNotFound)
+ return false;
+ throw std::runtime_error("AlignAccessAlignmentEnumeratorNext failed");
+ }
+ return false;
+ }
+ int position() const {
+ if (self) {
+ uint64_t p = 0;
+ AlignAccessAlignmentEnumeratorGetRefSeqPos(self, &p);
+ return (int)p;
+ }
+ return -1;
+ }
+ std::string SAM() const {
+ char buf[64 * 1024];
+ size_t sz = 0;
+ if (AlignAccessAlignmentEnumeratorGetSAM(self, &sz, sizeof(buf), buf) == 0) {
+ return std::string(buf, sz);
+ }
+ else {
+ return "";
+ }
+ }
+ };
+ class Database {
+ friend class Manager;
+ AlignAccessDB const *const self;
+ explicit Database(AlignAccessDB const *Self) : self(Self) {}
+ public:
+ Database(Database const &rhs) : self(rhs.self) {
+ rc_t const rc = AlignAccessDBAddRef(self);
+ if (rc) throw std::logic_error("AlignAccessDBAddRef failed");
+ }
+ ~Database() {
+ AlignAccessDBRelease(self);
+ }
+
+ AlignmentEnumerator slice(std::string const &refName, int start, int end) const
+ {
+ AlignAccessAlignmentEnumerator *p = 0;
+ rc_t const rc = AlignAccessDBWindowedAlignments(self, &p, refName.c_str(), start, end - start);
+ if (rc == 0) return AlignmentEnumerator(p);
+ AlignAccessAlignmentEnumeratorRelease(p);
+ if ((int)GetRCObject(rc) == rcRow && (int)GetRCState(rc) == rcNotFound)
+ return AlignmentEnumerator(0);
+ throw std::logic_error("AlignAccessDBWindowedAlignments failed");
+ }
+ };
+ class Manager {
+ AlignAccessMgr const *const self;
+ explicit Manager(AlignAccessMgr const *Self) : self(Self) {}
+ Manager() : self(0) {}
+ public:
+ ~Manager() {
+ AlignAccessMgrRelease(self);
+ }
+
+ Database open(std::string const &path, std::string const &indexPath) const {
+ VPath *dbp = 0;
+ VPath *idxp = 0;
+ rc_t rc = 0;
+ {
+ VFSManager *fsm = 0;
+
+ rc = VFSManagerMake(&fsm);
+ if (rc) throw std::logic_error("VFSManagerMake failed");
+
+ rc = VFSManagerMakeSysPath(fsm, &dbp, path.c_str());
+ if (rc) throw std::logic_error("VFSManagerMakeSysPath failed");
+
+ rc = VFSManagerMakeSysPath(fsm, &idxp, indexPath.c_str());
+ if (rc) throw std::logic_error("VFSManagerMakeSysPath failed");
+
+ VFSManagerRelease(fsm);
+ }
+ AlignAccessDB const *db = 0;
+ rc = AlignAccessMgrMakeIndexBAMDB(self, &db, dbp, idxp);
+ VPathRelease(dbp);
+ VPathRelease(idxp);
+ if (rc) throw std::runtime_error(std::string("failed to open ") + path + " with index " + indexPath);
+ return Database(db);
+ }
+
+ Database open(std::string const &path) const {
+ try {
+ return open(path, path + ".bai");
+ }
+ catch (std::runtime_error const &e) {}
+ catch (...) { throw; }
+
+ VPath *dbp = 0;
+ rc_t rc = 0;
+ {
+ VFSManager *fsm = 0;
+
+ rc = VFSManagerMake(&fsm);
+ if (rc) throw std::logic_error("VFSManagerMake failed");
+
+ rc = VFSManagerMakeSysPath(fsm, &dbp, path.c_str());
+ if (rc) throw std::logic_error("VFSManagerMakeSysPath failed");
+ VFSManagerRelease(fsm);
+ }
+ AlignAccessDB const *db = 0;
+ rc = AlignAccessMgrMakeBAMDB(self, &db, dbp);
+ VPathRelease(dbp);
+ if (rc) throw std::runtime_error(std::string("failed to open ") + path);
+ return Database(db);
+ }
+
+ static Manager make() {
+ AlignAccessMgr const *mgr = 0;
+ rc_t rc = AlignAccessMgrMake(&mgr);
+ if (rc != 0)
+ throw std::logic_error("AlignAccessMgrMake failed");
+ return Manager(mgr);
+ }
+ };
+};
+
diff --git a/libs/klib/release-vers.h b/test/align-access/main.c
similarity index 73%
copy from libs/klib/release-vers.h
copy to test/align-access/main.c
index 79ec52d..9136832 100644
--- a/libs/klib/release-vers.h
+++ b/test/align-access/main.c
@@ -24,21 +24,32 @@
*
*/
-/* THIS FILE IS NOT GENERATED AUTOMATICALLY! */
+#include <kapp/args.h>
+#include <klib/out.h>
+#include <kfg/config.h>
+extern int testMain(int, char **);
-/* Version of current SRA Toolkit Release */
-#define RELEASE_VERS 0x02080001
+ver_t CC KAppVersion ( void )
+{
+ return 0x1000000;
+}
+const char UsageDefaultName[] = "test-loader";
+
+rc_t CC UsageSummary (const char * progname)
+{
+ return KOutMsg ( "Usage:\n" "\t%s [options]\n\n", progname );
+}
+
+rc_t CC Usage( const Args* args )
+{
+ return 0;
+}
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ return testMain(argc, argv);
+}
-/* Type of Version of current SRA Toolkit Release is one of:
- * 'd' - development
- * 'a' - alpha
- * 'b' - beta
- * 'c' - release candidate
- * 'r' - final
- */
-#define RELEASE_TYPE 'r'
-/* Revision of Version of current SRA Toolkit Release */
-#define RELEASE_REVISION 0
diff --git a/test/align-access/test.cpp b/test/align-access/test.cpp
new file mode 100644
index 0000000..ce530ea
--- /dev/null
+++ b/test/align-access/test.cpp
@@ -0,0 +1,192 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Unit tests for the Loader module
+*/
+#include <ktst/unit_test.hpp>
+
+#include <klib/printf.h>
+
+#include <stdexcept>
+#include <string>
+#include <algorithm>
+
+#define VERY_VERBOSE (0)
+static bool const verbose = false;
+
+#include "align-access.hpp"
+
+using namespace std;
+
+TEST_SUITE(IndexTestSuite);
+
+struct Base
+{
+ struct TestSpec {
+ char const *ref;
+ int start;
+ int length;
+ int expected;
+ int result;
+ };
+ AlignAccess::Database const db;
+
+ int test1Range(char const *ref, int const start, int const length) const {
+ int alignments = 0;
+ AlignAccess::AlignmentEnumerator e = db.slice(ref, start - 1, (start - 1) + length);
+ while (e.next()) {
+ ++alignments;
+ }
+ return alignments;
+ }
+
+ Base(std::string const &file)
+ : db(AlignAccess::Manager::make().open(file))
+ {}
+ void testIndex(int const n, TestSpec test[]) const {
+ bool failed = false;
+ for (int i = 0; i < n; ++i)
+ test[i].result = test1Range(test[i].ref, test[i].start, test[i].length);
+ for (int i = 0; i < n; ++i) {
+ std::cout << "slice " << test[i].ref << ':' << test[i].start << '-' << (test[i].start - 1) + test[i].length << " should have " << test[i].expected << " alignments, got " << test[i].result << std::endl;
+ if (test[i].result != test[i].expected) {
+ failed = true;
+ }
+ }
+ if (failed)
+ throw std::runtime_error("failed");
+ }
+};
+
+class LoaderFixture1 : private Base
+{
+ static std::string BAM_FILE_NAME(void) {
+#if MAC
+ return std::string("/net/traces04/giab05/ftp/data/AshkenazimTrio/HG002_NA24385_son/NIST_HiSeq_HG002_Homogeneity-10953946/NHGRI_Illumina300X_AJtrio_novoalign_bams/HG002.GRCh38.300x.bam");
+#else
+ return std::string("/netmnt/traces04/giab05/ftp/data/AshkenazimTrio/HG002_NA24385_son/NIST_HiSeq_HG002_Homogeneity-10953946/NHGRI_Illumina300X_AJtrio_novoalign_bams/HG002.GRCh38.300x.bam");
+#endif
+ }
+public:
+ LoaderFixture1() : Base(BAM_FILE_NAME())
+ {
+ }
+ void testIndex(void) const {
+ Base::TestSpec test[] = {
+ { "chr1", 100000001, 1, 347, -1 },
+ { "chr1", 141484030, 11733, 0, -1 },
+ };
+ Base::testIndex(2, test);
+ }
+};
+
+FIXTURE_TEST_CASE ( LoadIndex1, LoaderFixture1 )
+{
+ testIndex();
+}
+
+class LoaderFixture2 : private Base
+{
+ static std::string BAM_FILE_NAME(void) {
+#if MAC
+ return std::string("/net/snowman/vol/projects/toolkit_test_data/traces04//1000genomes3/ftp/data/NA19240/exome_alignment/NA19240.mapped.SOLID.bfast.YRI.exome.20111114.bam");
+#else
+ return std::string("/net/snowman/vol/projects/toolkit_test_data/traces04//1000genomes3/ftp/data/NA19240/exome_alignment/NA19240.mapped.SOLID.bfast.YRI.exome.20111114.bam");
+#endif
+ }
+public:
+ LoaderFixture2() : Base(BAM_FILE_NAME())
+ {
+ }
+ void testIndex(void) const {
+ Base::TestSpec test[] = {
+ { "1", 1100001, 100000, 10387, -1 },
+ { "1", 1200001, 100000, 14957, -1 },
+ };
+ Base::testIndex(2, test);
+ }
+};
+
+FIXTURE_TEST_CASE ( LoadIndex2, LoaderFixture2 )
+{
+ testIndex();
+}
+
+class LoaderFixture3 : private Base
+{
+ static std::string BAM_FILE_NAME(void) {
+#if MAC
+ return std::string("/net/snowman/vol/projects/toolkit_test_data/traces04/1000genomes3/ftp/data/NA19240/exome_alignment/NA19240.mapped.SOLID.bfast.YRI.exome.20111114.bam");
+#else
+ return std::string("/netmnt/traces04/1kg_pilot_data/ftp/pilot_data/data/NA10851/alignment/NA10851.SLX.maq.SRP000031.2009_08.bam");
+#endif
+ }
+public:
+ LoaderFixture3() : Base(BAM_FILE_NAME())
+ {
+ }
+ void testIndex(void) const {
+ Base::TestSpec test[] = {
+ { "1", 24725087, 0, 33272242, -1 },
+ };
+ Base::testIndex(1, test);
+ }
+};
+
+FIXTURE_TEST_CASE ( LoadIndex3, LoaderFixture3 )
+{
+ testIndex();
+}
+
+static void test1Range(char const *path, char const *ref, int const start, int const end, bool const verbose)
+{
+ AlignAccess::Database const db(AlignAccess::Manager::make().open(path));
+ AlignAccess::AlignmentEnumerator e = db.slice(ref, start - 1, end);
+ int first = 0;
+ int last = 0;
+ int records = 0;
+ while (e.next()) {
+ last = e.position() + 1;
+ if (first == 0)
+ first = last;
+ if (verbose)
+ std::cout << "# " << e.SAM();
+ ++records;
+ }
+ std::cout << path << ' ' << ref << ':' << start << '-' << end << ' ' << records << ' ' << first << ' ' << last << std::endl;
+}
+
+extern "C" int testMain(int, char **);
+int testMain(int argc, char *argv[])
+{
+ if (argc >= 5) {
+ test1Range(argv[1], argv[2], atoi(argv[3]), atoi(argv[4]), argc > 5);
+ return 0;
+ }
+ return IndexTestSuite(argc, argv);
+}
+
diff --git a/test/align-access/test.pl b/test/align-access/test.pl
new file mode 100644
index 0000000..4e39ed4
--- /dev/null
+++ b/test/align-access/test.pl
@@ -0,0 +1,132 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+use Time::HiRes;
+use File::Spec;
+
+my $testfile = "/netmnt/traces04/giab05/ftp/data/AshkenazimTrio/HG002_NA24385_son/NIST_HiSeq_HG002_Homogeneity-10953946/NHGRI_Illumina300X_AJtrio_novoalign_bams/HG002.GRCh38.300x.bam";
+my $chrom = "chr1";
+my $chrLength = 248956422;
+
+chomp (my $samtools = `which samtools`);
+chomp (my $tester = `which test-align-access`);
+
+die "add samtools to PATH\n" unless $samtools;
+die "add test-align-access to PATH\n" unless $tester;
+die "can't read test file $testfile\n" unless -r $testfile;
+die "can't read test index $testfile.bai\n" unless -f "$testfile.bai";
+
+sub gold($$)
+{
+ exec {$samtools} $samtools, 'view', $testfile, "$chrom:$_[0]-$_[1]";
+ die "can't exec samtools";
+}
+
+sub lead($$)
+{
+ exec {$tester} $tester, $testfile, $chrom, @_;
+ die "can't exec $tester";
+}
+
+sub getGoldCount($$)
+{
+ my $kid = open(CHILD, '-|'); die "can't fork" unless defined $kid;
+ if ($kid == 0) {
+ goto &gold;
+ }
+ my $count = 0;
+ my ($first, $last);
+ while (defined(local $_ = <CHILD>)) {
+ my @F = \split(/\t/);
+ next if ((0+${$F[1]}) & 0x0004) != 0;
+ next if (''.${$F[2]}) eq '*';
+ next if (0+${$F[3]}) == 0;
+ next if (''.${$F[5]}) eq '*';
+ $last = 0+${$F[3]};
+ $first = $first || $last;
+ ++$count;
+ }
+ close CHILD;
+ return ($count, $first || '0', $last || '0');
+}
+
+sub getTestCountAndTime($$)
+{
+ my $kid = open(CHILD, '-|'); die "can't fork" unless defined $kid;
+ if ($kid == 0) {
+ goto &lead;
+ }
+ my $start = Time::HiRes::time;
+ chomp(my $out = <CHILD>);
+ my $end = Time::HiRes::time;
+ close CHILD;
+ die unless defined $out;
+ my ($count, $first, $last) = (split(/\s+/, $out))[-3..-1];
+ return ($count, $end - $start, $first, $last);
+}
+
+sub getGoldTime($$)
+{
+ my $kid = fork(); die "can't fork" unless defined $kid;
+ if ($kid == 0) {
+ open STDOUT, '>', File::Spec->devnull();
+ goto &gold;
+ }
+ my $start = Time::HiRes::time;
+ waitpid($kid, 0);
+ my $end = Time::HiRes::time;
+ die "samtools died with ".($?>>8)."\n" if $?;
+ return $end - $start;
+}
+
+my @tests;
+for (1..500) {
+ my $beg;
+ my $end;
+ do {
+ $beg = int(rand($chrLength));
+ $end = $beg + int(rand(100_000));
+ } while ($beg == $end || $end > $chrLength);
+ push @tests, { beg => $beg, end => $end };
+}
+my @timeDiff;
+
+for (sort { my $diff1 = $a->{beg} - $b->{beg}; return $diff1 == 0 ? $a->{end} - $b->{end} : $diff1 } @tests) {
+ my $beg = $_->{beg}+1;
+ my $end = $_->{end};
+ my ($goldCount, $goldFirst, $goldLast) = getGoldCount($beg, $end);
+ my ($testCount, $testTime, $testFirst, $testLast) = getTestCountAndTime($beg, $end);
+ if ($goldCount == $testCount && $goldFirst == $testFirst && $goldLast == $testLast) {
+ my $goldTime = getGoldTime($beg, $end);
+ push @timeDiff, $testTime / $goldTime;
+ next;
+ }
+ my $bad = ($goldFirst == $testFirst && $goldLast == $testLast) ? '' : '*';
+ print join("\t", $beg, $end, $goldCount, $testCount, $goldFirst, $testFirst, $goldLast, $testLast, $bad)."\n";
+}
+
+if (@timeDiff) {
+ my $N = scalar(@timeDiff);
+ my $sum = 0;
+ $sum += $_ for @timeDiff;
+
+ my $avg = $sum / $N;
+
+ $sum = 0;
+ $sum += ($_ - $avg) * ($_ - $avg) for @timeDiff;
+
+ my $stdev = sqrt($sum / $N);
+
+ printf "Average: %f; StDev: %f\n", $avg, $stdev;
+ for (@timeDiff) {
+ my $t;
+ if ($_ < $avg) {
+ $t = ($avg - $_) / $stdev;
+ }
+ else {
+ $t = ($_ - $avg) / $stdev;
+ }
+ print "$_\n" if $t > 2.0;
+ }
+}
diff --git a/test/install/.gitignore b/test/install/.gitignore
new file mode 100644
index 0000000..b64f634
--- /dev/null
+++ b/test/install/.gitignore
@@ -0,0 +1,2 @@
+install/
+root/
diff --git a/test/install/Makefile b/test/install/Makefile
new file mode 100644
index 0000000..fd91558
--- /dev/null
+++ b/test/install/Makefile
@@ -0,0 +1,139 @@
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+default: runtests
+
+TOP ?= $(abspath ../..)
+
+MODULE = test/install
+
+ALL_TARGS = \
+
+include $(TOP)/build/Makefile.env
+
+ROOTDIR = $(SRCDIR)/root
+
+SUBMAKE = $(MAKE) --no-print-directory
+
+ifeq (true,$(LINUX_ROOT))
+ #settings for the fake root mode
+ INSTALL_DIR = $(ROOTDIR)
+ KONFIG_DIR = $(ROOTDIR)/etc/ncbi
+ SUBMAKE += LINUX_ROOT=$(LINUX_ROOT) ROOT=$(INSTALL_DIR)
+else
+ INSTALL_DIR = $(SRCDIR)/install
+ KONFIG_DIR = $(INSTALL_DIR)/lib$(BITS)/ncbi
+endif
+
+# Override installation locations
+INST_BINDIR = $(INSTALL_DIR)/bin
+INST_LIBDIR = $(INSTALL_DIR)/lib$(BITS)
+INST_INCDIR = $(INSTALL_DIR)/include
+
+SUBMAKE += -C $(TOP) INST_BINDIR=$(INST_BINDIR) INST_LIBDIR=$(INSTALL_DIR)/lib INST_INCDIR=$(INST_INCDIR)
+
+#-------------------------------------------------------------------------------
+# runtests
+#
+
+INSTALLED_FILES = \
+ $(INST_LIBDIR)/libncbi-ngs-c++-static.a \
+ $(INST_LIBDIR)/libncbi-ngs-c++.a \
+ $(INST_LIBDIR)/libncbi-ngs-c++.a.$(MAJVERS) \
+ $(INST_LIBDIR)/libncbi-ngs-c++.a.$(VERSION) \
+ $(INST_LIBDIR)/libncbi-vdb-static.a \
+ $(INST_LIBDIR)/libncbi-vdb.a \
+ $(INST_LIBDIR)/libncbi-vdb.a.$(MAJVERS) \
+ $(INST_LIBDIR)/libncbi-vdb.a.$(VERSION) \
+ $(INST_LIBDIR)/libncbi-vdb.$(SHLX) \
+ $(INST_LIBDIR)/libncbi-vdb.$(MAJVERS_SHLX) \
+ $(INST_LIBDIR)/libncbi-vdb.$(VERSION_SHLX) \
+ $(INST_LIBDIR)/libncbi-wvdb-static.a \
+ $(INST_LIBDIR)/libncbi-wvdb.a \
+ $(INST_LIBDIR)/libncbi-wvdb.a.$(MAJVERS) \
+ $(INST_LIBDIR)/libncbi-wvdb.a.$(VERSION) \
+ $(INST_LIBDIR)/libncbi-wvdb.$(SHLX) \
+ $(INST_LIBDIR)/libncbi-wvdb.$(MAJVERS_SHLX) \
+ $(INST_LIBDIR)/libncbi-wvdb.$(VERSION_SHLX) \
+ $(KONFIG_DIR)/certs.kfg \
+ $(KONFIG_DIR)/default.kfg \
+ $(INST_INCDIR)/ncbi-vdb/NGS.hpp \
+
+
+runtests: test-install
+
+test-install: cleanup
+ @ echo "installing to $(INSTALL_DIR)"
+ @ #
+ @ echo "1. clean install"
+ @ #
+ @ $(SUBMAKE) install
+ @ for i in $(INSTALLED_FILES); do test -f "$$i" || ( echo "$$i is not found" >>$(INST_INCDIR)/missing ); done
+ @ ! test -f $(INST_INCDIR)/missing || ! cat $(INST_INCDIR)/missing
+ @ #
+ @ echo "2. uninstall"
+ @ #
+ @ $(SUBMAKE) uninstall
+ @ for i in $(INSTALLED_FILES); do ! test -f "$$i" || ( echo "$$i still exists" >>$(INST_INCDIR)/exist ); done
+ @ ! test -f $(INST_INCDIR)/exist || ! cat $(INST_INCDIR)/exist
+ @ #
+ @ echo "3. re-install, preserving modified .kfg files"
+ @ #
+ @ # install
+ @ $(SUBMAKE) install
+ @ # modify .kfg files
+ @ echo newcert >$(KONFIG_DIR)/certs.kfg
+ @ echo newdflt >$(KONFIG_DIR)/default.kfg
+ @ echo newncbi >$(KONFIG_DIR)/ncbi-vdb.kfg # this should be renamed and never installed again
+ @ # reinstall
+ @ $(SUBMAKE) install
+ @ # verify changed files have been saved
+ @ grep newcert $(KONFIG_DIR)/certs.kfg.orig >/dev/null
+ @ grep newdflt $(KONFIG_DIR)/default.kfg.orig >/dev/null
+ @ grep newncbi $(KONFIG_DIR)/ncbi-vdb.kfg.orig >/dev/null
+ @ ! test -f $(KONFIG_DIR)/ncbi-vdb.kfg # gone ...
+ @ # verify that further reinstall does not overwrite previously saved files if there is no change
+ @ $(SUBMAKE) install
+ @ # verify saved files are still there
+ @ grep newcert $(KONFIG_DIR)/certs.kfg.orig >/dev/null
+ @ grep newdflt $(KONFIG_DIR)/default.kfg.orig >/dev/null
+ @ grep newncbi $(KONFIG_DIR)/ncbi-vdb.kfg.orig >/dev/null
+ @ ! test -f $(KONFIG_DIR)/ncbi-vdb.kfg # ... and stays gone
+ @ #
+ @ #
+ifeq (,$(LINUX_ROOT))
+ #
+ # Run the same tests in simulated root mode
+ #
+ @ $(MAKE) LINUX_ROOT=true
+endif
+
+cleanup:
+ rm -rf $(INSTALL_DIR)
+ifeq (true,$(LINUX_ROOT))
+ # in root mode, recreate "system" directories
+ mkdir -p $(ROOTDIR)/usr/include $(ROOTDIR)/etc/profile.d
+endif
+
+.PHONY: install
\ No newline at end of file
diff --git a/test/kdb/kdbtest.cpp b/test/kdb/kdbtest.cpp
index 3ba1646..56e181d 100644
--- a/test/kdb/kdbtest.cpp
+++ b/test/kdb/kdbtest.cpp
@@ -60,9 +60,9 @@ TEST_CASE(KDBManagerVPathType)
const struct KFile *dummy1;
const struct VPath *dummy2;
REQUIRE_RC(VFSManagerResolveSpec ( vfsmgr, "SRR000123", &vpath, &dummy1, &dummy2, true));
-
+
path = ToString(vpath);
-
+
REQUIRE_RC(KFileRelease(dummy1));
REQUIRE_RC(VPathRelease(dummy2));
REQUIRE_RC(VPathRelease(vpath));
@@ -76,9 +76,9 @@ TEST_CASE(KDBManagerVPathType)
//cout << path << endl;
REQUIRE_EQ((int)kptTable, KDBManagerPathType(mgr, path.c_str()));
-
+
REQUIRE_RC(KDBManagerRelease(mgr));
-
+
}
// KColumnBlob
@@ -93,15 +93,15 @@ public:
{
const KDBManager* mgr;
THROW_ON_RC ( KDBManagerMakeRead ( & mgr, NULL ) );
-
+
const KTable* tbl;
THROW_ON_RC ( KDBManagerOpenTableRead ( mgr, & tbl, "SRR000123" ) );
-
+
const KColumn* col;
THROW_ON_RC ( KTableOpenColumnRead ( tbl, & col, "X" ) );
-
+
THROW_ON_RC ( KColumnOpenBlobRead ( col, & m_blob, 1 ) );
-
+
THROW_ON_RC ( KColumnRelease ( col ) );
THROW_ON_RC ( KTableRelease ( tbl ) );
THROW_ON_RC ( KDBManagerRelease ( mgr ) );
@@ -117,9 +117,9 @@ public:
};
FIXTURE_TEST_CASE ( ColumnBlobRead_basic, ColumnBlobReadFixture )
-{
- const size_t BlobSize = 1882;
- const size_t BufSize = 2024;
+{
+ const size_t BlobSize = 1882;
+ const size_t BufSize = 2024;
char buffer [ BufSize ];
REQUIRE_RC ( KColumnBlobRead ( m_blob, 0, buffer, BufSize, & m_num_read, & m_remaining ) );
REQUIRE_EQ ( BlobSize, m_num_read );
@@ -127,9 +127,9 @@ FIXTURE_TEST_CASE ( ColumnBlobRead_basic, ColumnBlobReadFixture )
}
FIXTURE_TEST_CASE ( ColumnBlobRead_insufficient_buffer, ColumnBlobReadFixture )
-{
- const size_t BlobSize = 1882;
- const size_t BufSize = 1024;
+{
+ const size_t BlobSize = 1882;
+ const size_t BufSize = 1024;
char buffer [ BufSize ];
// first read incomplete
REQUIRE_RC ( KColumnBlobRead ( m_blob, 0, buffer, BufSize, & m_num_read, & m_remaining ) );
@@ -174,7 +174,8 @@ const char UsageDefaultName[] = "test-kdb";
rc_t CC KMain ( int argc, char *argv [] )
{
//assert(!KDbgSetString("KFG"));
- //assert(!KDbgSetString("VFS"));
+ assert(!KDbgSetString("VFS"));
+ //KDbgSetModConds ( DBG_KNS, DBG_FLAG ( DBG_KNS_SOCKET ), DBG_FLAG ( DBG_KNS_SOCKET ) );
KConfigDisableUserSettings();
rc_t rc=KdbTestSuite(argc, argv);
return rc;
diff --git a/test/kfg/flat-sra-kfg.cpp b/test/kfg/flat-sra-kfg.cpp
index 979dec2..2959432 100644
--- a/test/kfg/flat-sra-kfg.cpp
+++ b/test/kfg/flat-sra-kfg.cpp
@@ -127,9 +127,9 @@ static const char badCgi[]
#ifdef ALL
TEST_CASE(test_sra) {
const string newShort
- ("https://sra-download.ncbi.nlm.nih.gov/srapub/SRR000001");
+ ("https://sra-download.ncbi.nlm.nih.gov/traces/sra27/SRR/000000/SRR000001");
const string newLong
- ("https://sra-download.ncbi.nlm.nih.gov/srapub/SRR1000254");
+ ("https://sra-download.ncbi.nlm.nih.gov/traces/sra14/SRR/000976/SRR1000254");
const string oldShort("https://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/"
"reads/ByRun/sra/SRR/SRR000/SRR000001/SRR000001.sra");
diff --git a/test/kfg/keystoretest.cpp b/test/kfg/keystoretest.cpp
index 728d8f8..ec828a0 100644
--- a/test/kfg/keystoretest.cpp
+++ b/test/kfg/keystoretest.cpp
@@ -171,30 +171,30 @@ FIXTURE_TEST_CASE(KeyStoreGetKey_Protected, KeyStoreFixture)
REQUIRE_RC(KDirectoryRemove(wd, true, GetName()));
}
-FIXTURE_TEST_CASE(KeyStoreGetKeyById_Protected, KeyStoreFixture)
-{
- const char tempKey[] = "another tempkey from file";
- {
- ofstream f(GetName());
- f << tempKey << endl;
- }
-
- KfgUpdateNode("/repository/user/protected/dbGaP-2956/root", ".");
- KfgUpdateNode("/repository/user/protected/dbGaP-2956/encryption-key-path",
- "wrong file!");
- KfgUpdateNode("/repository/user/protected/dbGaP-2957/root", ".");
- KfgUpdateNode("/repository/user/protected/dbGaP-2957/encryption-key-path",
- GetName());
-
- REQUIRE_RC(KKeyStoreSetConfig(ks, kfg));
-
- REQUIRE_RC(KKeyStoreGetKeyByProjectId(ks,
- "give us the key for 2957", &key, 2957));
- REQUIRE_NOT_NULL(key);
- REQUIRE_EQ(string(tempKey), string(key->value.addr, key->value.len));
-
- REQUIRE_RC(KDirectoryRemove(wd, true, GetName()));
-}
+FIXTURE_TEST_CASE(KeyStoreGetKeyById_Protected, KeyStoreFixture)
+{
+ const char tempKey[] = "another tempkey from file";
+ {
+ ofstream f(GetName());
+ f << tempKey << endl;
+ }
+
+ KfgUpdateNode("/repository/user/protected/dbGaP-2956/root", ".");
+ KfgUpdateNode("/repository/user/protected/dbGaP-2956/encryption-key-path",
+ "wrong file!");
+ KfgUpdateNode("/repository/user/protected/dbGaP-2957/root", ".");
+ KfgUpdateNode("/repository/user/protected/dbGaP-2957/encryption-key-path",
+ GetName());
+
+ REQUIRE_RC(KKeyStoreSetConfig(ks, kfg));
+
+ REQUIRE_RC(KKeyStoreGetKeyByProjectId(ks,
+ "give us the key for 2957", &key, 2957));
+ REQUIRE_NOT_NULL(key);
+ REQUIRE_EQ(string(tempKey), string(key->value.addr, key->value.len));
+
+ REQUIRE_RC(KDirectoryRemove(wd, true, GetName()));
+}
//
// Object Id / Object name bindings
diff --git a/test/kfg/kfgtest.cpp b/test/kfg/kfgtest.cpp
index bb37b1c..cc14e82 100644
--- a/test/kfg/kfgtest.cpp
+++ b/test/kfg/kfgtest.cpp
@@ -131,8 +131,9 @@ public:
file=0;
}
- void CreateAndLoad(const char* name, const char* contents)
+ void CreateAndLoad(const string & sname, const char* contents)
{
+ const char * name = sname . c_str ();
#ifdef DBG_KFG
cout << "26 CreateAndLoad(" << name << ")\n";
#endif
diff --git a/test/kfs/Makefile b/test/kfs/Makefile
index 47ea200..aaa07d9 100644
--- a/test/kfs/Makefile
+++ b/test/kfs/Makefile
@@ -30,10 +30,11 @@ TOP ?= $(abspath ../..)
MODULE = test/kfs
TEST_TOOLS = \
+ kdf \
test-kfs \
- test-ramfile \
- test-resolve \
- test-cachetee \
+ test-ramfile \
+ test-resolve \
+ test-cachetee \
test-kfs-md5
include $(TOP)/build/Makefile.env
@@ -67,6 +68,20 @@ endif
$(TEST_BINDIR)/test-kfs: $(TEST_KFS_OBJ)
$(LP) --exe -o $@ $^ $(TEST_KFS_LIB)
+
+#-------------------------------------------------------------------------------
+# df
+#
+DU_SRC = \
+ kdf
+
+DU_OBJ = \
+ $(addsuffix .$(OBJX),$(DU_SRC))
+
+$(TEST_BINDIR)/kdf: $(DU_OBJ)
+ $(LP) --exe -o $@ $^ $(TEST_KFS_LIB)
+
+
#-------------------------------------------------------------------------------
# test-kfs-md5
#
@@ -135,3 +150,52 @@ TEST_CACHETEE_LIB = \
$(TEST_BINDIR)/test-cachetee: $(TEST_CACHETEE_OBJ)
$(LP) --exe -o $@ $^ $(TEST_CACHETEE_LIB)
+ rm *.o *.d
+
+#-------------------------------------------------------------------------------
+# fuse-proxy ( needed for the cachetee_out_of_space - test
+#
+FUSE_PROXY_SRC = \
+ fuse_proxy
+
+FUSE_PROXY_OBJ = \
+ $(addsuffix .$(OBJX),$(FUSE_PROXY_SRC))
+
+FUSE_PROXY_LIB = \
+ -lfuse
+
+$(TEST_BINDIR)/fuse-proxy: $(FUSE_PROXY_OBJ)
+ $(LP) --exe -o $@ $^ $(FUSE_PROXY_LIB)
+ rm *.o *.d
+
+#-------------------------------------------------------------------------------
+# cachetee_out_of_space
+#
+CTOFS_SRC = \
+ cachetee-out-of-space
+
+CTOFS_OBJ = \
+ $(addsuffix .$(OBJX),$(CTOFS_SRC))
+
+CTOFS_LIB = \
+ -lkapp \
+ -sncbi-vdb \
+ -lktst
+
+$(TEST_BINDIR)/test-cachetee-out-of-space: $(CTOFS_OBJ)
+ $(LP) --exe -o $@ $^ $(CTOFS_LIB)
+ rm *.o *.d
+
+ifeq (linux,$(OS))
+ifeq (1,$(HAVE_FUSE))
+#-------------------------------------------------------------------------------
+# scripted tests
+#
+runtests: cachetee-out-of-space-test
+
+cachetee-out-of-space-test: $(TEST_BINDIR)/fuse-proxy $(TEST_BINDIR)/test-cachetee-out-of-space run-cachetee-out-of-space-test.sh
+ @ echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++"
+ @ echo "Running $(SRCDIR)/run-cachetee-out-of-space-test.sh"
+ @ $(SRCDIR)/run-cachetee-out-of-space-test.sh $(SRCDIR) $(TEST_BINDIR)
+endif
+endif
diff --git a/test/kfs/cachetee-out-of-space.cpp b/test/kfs/cachetee-out-of-space.cpp
new file mode 100644
index 0000000..2a8cb23
--- /dev/null
+++ b/test/kfs/cachetee-out-of-space.cpp
@@ -0,0 +1,187 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <cstring>
+#include <cstdlib>
+
+#include <ktst/unit_test.hpp>
+
+#include <klib/out.h>
+#include <klib/rc.h>
+
+#include <kfs/defs.h>
+#include <kfs/directory.h>
+#include <kfs/file.h>
+#include <kfs/cacheteefile.h>
+
+using namespace std;
+
+#define DATAFILE "org.dat"
+#define DATAFILESIZE ( 1024 * 12 )
+#define CACHEFILE "mount_point/cache.dat"
+#define BLOCKSIZE 128
+
+TEST_SUITE( CacheTeeOutOfSpaceSuite );
+
+static uint32_t rand_32( uint32_t min, uint32_t max )
+{
+ double scaled = ( ( double )rand() / RAND_MAX );
+ return ( ( max - min + 1 ) * scaled ) + min;
+}
+
+static rc_t fill_file_with_random_data( KFile * file, size_t file_size )
+{
+ rc_t rc = KFileSetSize( file, file_size );
+ if ( rc == 0 )
+ {
+ uint64_t pos = 0;
+ size_t total = 0;
+ while ( rc == 0 && total < file_size )
+ {
+ uint32_t data[ 512 ];
+ uint32_t i;
+ size_t to_write, num_writ;
+
+ for ( i = 0; i < 512; ++i ) data[ i ] = rand_32( 0, 0xFFFFFFFF - 1 );
+ to_write = ( file_size - total );
+ if ( to_write > sizeof data ) to_write = sizeof data;
+ rc = KFileWriteAll ( file, pos, data, to_write, &num_writ );
+ if ( rc == 0 )
+ {
+ pos += num_writ;
+ total += num_writ;
+ }
+ }
+ }
+ return rc;
+}
+
+static rc_t create_random_file( KDirectory * dir, const char * filename, uint64_t file_size )
+{
+ KFile * file;
+ rc_t rc = KDirectoryCreateFile ( dir, &file, false, 0664, kcmInit, filename );
+ if ( rc == 0 )
+ {
+ if ( rc == 0 )
+ rc = fill_file_with_random_data( file, file_size );
+ KFileRelease( file );
+ }
+ return rc;
+}
+
+
+static rc_t read_all( const KFile * src, size_t block_size )
+{
+ rc_t rc = 0;
+ uint8_t * buffer = ( uint8_t * )malloc( block_size );
+ if ( buffer == NULL )
+ rc = RC ( rcRuntime, rcBuffer, rcConstructing, rcMemory, rcExhausted );
+ else
+ {
+ uint64_t pos = 0;
+ size_t num_read = 1;
+ while ( rc == 0 && num_read > 0 )
+ {
+ rc = KFileReadAll ( src, pos, buffer, block_size, &num_read );
+ if ( rc == 0 )
+ pos += num_read;
+ else
+ {
+ KOutMsg( "KFileReadAll( pos: %ld, block_size: %ld ) -> num_read: %ld, %R\n",
+ pos, block_size, num_read, rc );
+ }
+ }
+ free( buffer );
+ rc = 0;
+ }
+ return rc;
+}
+
+
+static rc_t remove_file( KDirectory * dir, const char * filename )
+{
+ return KDirectoryRemove ( dir, true, "%s", filename );
+}
+
+
+//////////////////////////////////////////// Test-cases
+
+TEST_CASE( CacheTee_out_of_space )
+{
+ KOutMsg( "Test: CacheTee_out_of_space\n" );
+
+ KDirectory * dir;
+ REQUIRE_RC( KDirectoryNativeDir( &dir ) );
+
+ // we are creating a file filled with random data, to stand in as the remote file
+ // which has to be cached...
+ REQUIRE_RC( create_random_file( dir, DATAFILE, DATAFILESIZE ) );
+
+ // we open this file with random-data, created above
+ const KFile * org;
+ REQUIRE_RC( KDirectoryOpenFileRead( dir, &org, "%s", DATAFILE ) );
+
+ // we wrap this data-file into a cache-file
+ const KFile * tee;
+ REQUIRE_RC( KDirectoryMakeCacheTee ( dir, &tee, org, BLOCKSIZE, "%s", CACHEFILE ) );
+
+ // now we can release the original data-file, the tee-file holds a reference to it...
+ REQUIRE_RC( KFileRelease( org ) );
+
+ // this is the part that should not fail: we are reading the whole content of the tee-file.
+ // because we have created the tee-file in a directory that has a quota...
+ // the tee-file cannot grow to the necessary size, it should internally switch into
+ // a passtrough-mode and just read the org-file instead of trying to cache...
+ REQUIRE_RC( read_all( tee, ( 1024 * 7 ) ) );
+
+ // we clean up, by releasing the tee-file, and removing all the temp. files we created
+ REQUIRE_RC( KFileRelease( tee ) );
+ REQUIRE_RC( remove_file( dir, CACHEFILE ) );
+ REQUIRE_RC( remove_file( dir, DATAFILE ) );
+ REQUIRE_RC( KDirectoryRelease( dir ) );
+}
+
+
+//////////////////////////////////////////// Main
+extern "C"
+{
+
+#include <kapp/args.h>
+#include <kfg/config.h>
+
+ ver_t CC KAppVersion ( void ) { return 0x1000000; }
+ rc_t CC UsageSummary ( const char * progname ) { return 0; }
+ rc_t CC Usage ( const Args * args ) { return 0; }
+ const char UsageDefaultName[] = "test-cachetee-out-of-space";
+
+ rc_t CC KMain ( int argc, char *argv [] )
+ {
+ KConfigDisableUserSettings();
+ return CacheTeeOutOfSpaceSuite( argc, argv );
+ }
+
+}
+
diff --git a/test/kfs/cacheteetest.cpp b/test/kfs/cacheteetest.cpp
index be2eab0..f5a1015 100644
--- a/test/kfs/cacheteetest.cpp
+++ b/test/kfs/cacheteetest.cpp
@@ -588,7 +588,7 @@ const char UsageDefaultName[] = "cachetee-test";
rc_t CC KMain ( int argc, char *argv [] )
{
- srand(time(NULL));
+ srand( time( NULL ) );
KConfigDisableUserSettings();
rc_t rc = prepare_cachetee_tests();
if ( rc == 0 )
@@ -596,7 +596,7 @@ rc_t CC KMain ( int argc, char *argv [] )
rc = CacheTeeTests( argc, argv );
finish_cachetee_tests();
}
- KOutMsg( "and the result is: %d", rc );
+ KOutMsg( "and the result is: %R\n", rc );
return rc;
}
diff --git a/test/kfs/fuse_proxy.c b/test/kfs/fuse_proxy.c
new file mode 100644
index 0000000..f2fedb1
--- /dev/null
+++ b/test/kfs/fuse_proxy.c
@@ -0,0 +1,668 @@
+/*
+ FUSE: Filesystem in Userspace
+ Copyright (C) 2001-2007 Miklos Szeredi <miklos at szeredi.hu>
+ Copyright (C) 2011 Sebastian Pipping <sebastian at pipping.org>
+
+ This program can be distributed under the terms of the GNU GPL.
+ See the file COPYING.
+
+ gcc -Wall fusexmp.c `pkg-config fuse --cflags --libs` -o fusexmp
+*/
+
+#define FUSE_USE_VERSION 26
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef linux
+/* For pread()/pwrite()/utimensat() */
+#define _XOPEN_SOURCE 700
+#endif
+
+#include <fuse.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <errno.h>
+#include <sys/time.h>
+#ifdef HAVE_SETXATTR
+#include <sys/xattr.h>
+#endif
+
+#define PREFIX_SIZE 4096
+char prefix[ PREFIX_SIZE ];
+int prefix_len;
+off_t prefix_truncate_limit = 0;
+off_t prefix_write_limit = 0;
+
+static void set_prefix_truncate_limit( off_t new_limit )
+{
+ prefix_truncate_limit = new_limit;
+}
+
+static bool above_truncate_limit( off_t value )
+{
+ return ( ( prefix_truncate_limit > 0 ) && ( value >= prefix_truncate_limit ) );
+}
+
+static void set_prefix_write_limit( off_t new_limit )
+{
+ prefix_write_limit = new_limit;
+}
+
+static bool above_write_limit( off_t value )
+{
+ return ( ( prefix_write_limit > 0 ) && ( value >= prefix_write_limit ) );
+}
+
+static int set_prefix( const char * new_prefix )
+{
+ char buffer[ PREFIX_SIZE ];
+ char *abs_path = realpath( new_prefix, buffer );
+
+ prefix[ 0 ] = 0;
+ prefix_len = 0;
+ if ( abs_path != NULL )
+ {
+ int new_prefix_len = strlen( abs_path );
+ if ( new_prefix_len < PREFIX_SIZE )
+ {
+ strncpy( prefix, new_prefix, new_prefix_len );
+ prefix[ new_prefix_len ] = 0;
+ prefix_len = new_prefix_len;
+ }
+ }
+ return prefix_len;
+}
+
+static const char * make_prefixed_path( const char * path )
+{
+ int path_len = strlen( path );
+ int prefixed_len = prefix_len + path_len;
+ char * res = malloc( prefixed_len + 1 );
+ strncpy( res, prefix, prefix_len );
+ strncpy( &( res[ prefix_len ] ), path, path_len );
+ res[ prefixed_len ] = 0;
+ return res;
+}
+
+static void release_prefixed_path( const char * path )
+{
+ free( ( void * ) path );
+}
+
+static size_t str_2_size( const char * s )
+{
+ size_t res = 0;
+ if ( s != NULL )
+ {
+ size_t l = strlen( s );
+ if ( l > 0 )
+ {
+ size_t multipl = 1;
+ switch( s[ l - 1 ] )
+ {
+ case 'k' :
+ case 'K' : multipl = 1024; break;
+ case 'm' :
+ case 'M' : multipl = 1024 * 1024; break;
+ case 'g' :
+ case 'G' : multipl = 1024 * 1024 * 1024; break;
+ }
+
+ {
+ char * endptr;
+ res = strtol( s, &endptr, 0 ) * multipl;
+ }
+ }
+ }
+ return res;
+}
+
+static size_t used_space_of_file( const char * path )
+{
+ size_t res = 0;
+ struct stat file_status;
+ int stat_res = stat( path, &file_status );
+ if ( stat_res == 0 )
+ res = ( file_status.st_blocks * 512 );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_getattr( const char *path, struct stat *stbuf )
+{
+ int res = lstat( path, stbuf );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_getattr( const char *path, struct stat *stbuf )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_getattr( prefixed_path, stbuf );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_access( const char * path, int mask )
+{
+ int res = access( path, mask );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_access( const char * path, int mask )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_access( prefixed_path, mask );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_readlink( const char *path, char *buf, size_t size )
+{
+ int res = readlink( path, buf, size - 1 );
+ if ( res == -1 ) return -errno;
+ buf[ res ] = '\0';
+ return 0;
+}
+
+static int xmp_prefixed_readlink( const char *path, char *buf, size_t size )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_readlink( prefixed_path, buf, size );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_readdir( const char *path, void *buf, fuse_fill_dir_t filler,
+ off_t offset, struct fuse_file_info *fi )
+{
+ DIR *dp;
+ struct dirent *de;
+
+ (void) offset;
+ (void) fi;
+
+ dp = opendir( path );
+ if ( dp == NULL ) return -errno;
+
+ while ( ( de = readdir( dp ) ) != NULL )
+ {
+ struct stat st;
+ memset( &st, 0, sizeof( st ) );
+ st.st_ino = de->d_ino;
+ st.st_mode = de->d_type << 12;
+ if ( filler( buf, de->d_name, &st, 0 ) )
+ break;
+ }
+
+ closedir( dp );
+ return 0;
+}
+
+static int xmp_prefixed_readdir( const char *path, void *buf, fuse_fill_dir_t filler,
+ off_t offset, struct fuse_file_info *fi )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_readdir( prefixed_path, buf, filler, offset, fi );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_mknod( const char *path, mode_t mode, dev_t rdev )
+{
+ int res;
+
+ /* On Linux this could just be 'mknod(path, mode, rdev)' but this
+ is more portable */
+ if ( S_ISREG( mode ) )
+ {
+ res = open( path, O_CREAT | O_EXCL | O_WRONLY, mode );
+ if ( res >= 0 ) res = close( res );
+ }
+ else if ( S_ISFIFO( mode ) )
+ {
+ res = mkfifo( path, mode );
+ }
+ else
+ {
+ res = mknod( path, mode, rdev );
+ }
+ if ( res == -1 ) return -errno;
+
+ return 0;
+}
+
+static int xmp_prefixed_mknod( const char *path, mode_t mode, dev_t rdev )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_mknod( prefixed_path, mode, rdev );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_mkdir( const char *path, mode_t mode )
+{
+ int res;
+ res = mkdir( path, mode );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_mkdir( const char *path, mode_t mode )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_mkdir( prefixed_path, mode );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_unlink( const char *path )
+{
+ int res;
+ res = unlink( path );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_unlink( const char *path )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_unlink( prefixed_path );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_rmdir( const char *path )
+{
+ int res;
+ res = rmdir( path );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_rmdir( const char *path )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_rmdir( prefixed_path );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_symlink( const char *from, const char *to )
+{
+ int res;
+ res = symlink( from, to );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_symlink( const char *from, const char *to )
+{
+ const char *prefixed_from = make_prefixed_path( from );
+ const char *prefixed_to = make_prefixed_path( to );
+ int res = xmp_symlink( prefixed_from, prefixed_to );
+ release_prefixed_path( prefixed_to );
+ release_prefixed_path( prefixed_from );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_rename( const char *from, const char *to )
+{
+ int res;
+ res = rename( from, to );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_rename( const char *from, const char *to )
+{
+ const char *prefixed_from = make_prefixed_path( from );
+ const char *prefixed_to = make_prefixed_path( to );
+ int res = xmp_rename( prefixed_from, prefixed_to );
+ release_prefixed_path( prefixed_to );
+ release_prefixed_path( prefixed_from );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_link( const char *from, const char *to )
+{
+ int res = link( from, to );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_link( const char *from, const char *to )
+{
+ const char *prefixed_from = make_prefixed_path( from );
+ const char *prefixed_to = make_prefixed_path( to );
+ int res = xmp_link( prefixed_from, prefixed_to );
+ release_prefixed_path( prefixed_to );
+ release_prefixed_path( prefixed_from );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_chmod( const char *path, mode_t mode )
+{
+ int res = chmod( path, mode );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_chmod( const char *path, mode_t mode )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_chmod( prefixed_path, mode );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_chown( const char *path, uid_t uid, gid_t gid )
+{
+ int res = lchown( path, uid, gid );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_chown( const char *path, uid_t uid, gid_t gid )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_chown( prefixed_path, uid, gid );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_truncate( const char *path, off_t size )
+{
+ int res = truncate( path, size );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_truncate( const char *path, off_t size )
+{
+ int res = 0;
+ if ( above_truncate_limit( size ) )
+ res = -EFBIG;
+ else
+ {
+ const char *prefixed_path = make_prefixed_path( path );
+ res = xmp_truncate( prefixed_path, size );
+ release_prefixed_path( prefixed_path );
+ }
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+#ifdef HAVE_UTIMENSAT
+static int xmp_utimens( const char *path, const struct timespec ts[2] )
+{
+ /* don't use utime/utimes since they follow symlinks */
+ int res = utimensat( 0, path, ts, AT_SYMLINK_NOFOLLOW );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_utimens( const char *path, const struct timespec ts[2] )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_utimens( prefixed_path, size );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+#endif
+
+/* -------------------------------------------------------------------------- */
+static int xmp_open( const char *path, struct fuse_file_info *fi )
+{
+ int res = open( path, fi->flags );
+ if ( res == -1 ) return -errno;
+ close( res );
+ return 0;
+}
+
+static int xmp_prefixed_open( const char *path, struct fuse_file_info *fi )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_open( prefixed_path, fi );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_read( const char *path, char *buf, size_t size, off_t offset,
+ struct fuse_file_info *fi )
+{
+ int fd;
+ int res;
+
+ ( void ) fi;
+ fd = open( path, O_RDONLY );
+ if ( fd == -1 ) return -errno;
+
+ res = pread( fd, buf, size, offset );
+ if ( res == -1 ) res = -errno;
+ close( fd );
+
+ return res;
+}
+
+static int xmp_prefixed_read( const char *path, char *buf, size_t size, off_t offset,
+ struct fuse_file_info *fi )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_read( prefixed_path, buf, size, offset, fi );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_write( const char *path, const char *buf, size_t size,
+ off_t offset, struct fuse_file_info *fi )
+{
+ int res;
+
+ ( void ) fi;
+ int fd = open( path, O_WRONLY );
+ if ( fd == -1 ) return -errno;
+
+ res = pwrite( fd, buf, size, offset );
+ if ( res == -1 ) res = -errno;
+
+ close( fd );
+ return res;
+}
+
+static int xmp_prefixed_write( const char *path, const char *buf, size_t size,
+ off_t offset, struct fuse_file_info *fi )
+{
+ int res;
+ const char *prefixed_path = make_prefixed_path( path );
+ if ( above_write_limit( used_space_of_file( prefixed_path ) ) )
+ res = -EFBIG;
+ else
+ res = xmp_write( prefixed_path, buf, size, offset, fi );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_statfs( const char *path, struct statvfs *stbuf )
+{
+ int res = statvfs( path, stbuf );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_prefixed_statfs( const char *path, struct statvfs *stbuf )
+{
+ const char *prefixed_path = make_prefixed_path( path );
+ int res = xmp_statfs( prefixed_path, stbuf );
+ release_prefixed_path( prefixed_path );
+ return res;
+}
+
+/* -------------------------------------------------------------------------- */
+static int xmp_release( const char *path, struct fuse_file_info *fi )
+{
+ /* Just a stub. This method is optional and can safely be left
+ unimplemented */
+
+ (void) path;
+ (void) fi;
+ return 0;
+}
+
+static int xmp_fsync( const char *path, int isdatasync,
+ struct fuse_file_info *fi )
+{
+ /* Just a stub. This method is optional and can safely be left
+ unimplemented */
+
+ (void) path;
+ (void) isdatasync;
+ (void) fi;
+ return 0;
+}
+
+#ifdef HAVE_POSIX_FALLOCATE
+static int xmp_fallocate( const char *path, int mode,
+ off_t offset, off_t length, struct fuse_file_info *fi )
+{
+ int fd;
+ int res;
+
+ ( void ) fi;
+
+ if ( mode ) return -EOPNOTSUPP;
+
+ fd = open( path, O_WRONLY );
+ if ( fd == -1 ) return -errno;
+
+ res = -posix_fallocate( fd, offset, length );
+
+ close( fd );
+ return res;
+}
+#endif
+
+#ifdef HAVE_SETXATTR
+/* xattr operations are optional and can safely be left unimplemented */
+static int xmp_setxattr( const char *path, const char *name, const char *value,
+ size_t size, int flags )
+{
+ int res = lsetxattr( path, name, value, size, flags );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+
+static int xmp_getxattr( const char *path, const char *name, char *value, size_t size )
+{
+ int res = lgetxattr( path, name, value, size );
+ if ( res == -1 ) return -errno;
+ return res;
+}
+
+static int xmp_listxattr( const char *path, char *list, size_t size )
+{
+ int res = llistxattr( path, list, size );
+ if ( res == -1 ) return -errno;
+ return res;
+}
+
+static int xmp_removexattr( const char *path, const char *name )
+{
+ int res = lremovexattr( path, name );
+ if ( res == -1 ) return -errno;
+ return 0;
+}
+#endif /* HAVE_SETXATTR */
+
+static struct fuse_operations xmp_oper = {
+ .getattr = xmp_prefixed_getattr,
+ .access = xmp_prefixed_access,
+ .readlink = xmp_prefixed_readlink,
+ .readdir = xmp_prefixed_readdir,
+ .mknod = xmp_prefixed_mknod,
+ .mkdir = xmp_prefixed_mkdir,
+ .symlink = xmp_prefixed_symlink,
+ .unlink = xmp_prefixed_unlink,
+ .rmdir = xmp_prefixed_rmdir,
+ .rename = xmp_prefixed_rename,
+ .link = xmp_prefixed_link,
+ .chmod = xmp_prefixed_chmod,
+ .chown = xmp_prefixed_chown,
+ .truncate = xmp_prefixed_truncate,
+#ifdef HAVE_UTIMENSAT
+ .utimens = xmp_prefixed_utimens,
+#endif
+ .open = xmp_prefixed_open,
+ .read = xmp_prefixed_read,
+ .write = xmp_prefixed_write,
+ .statfs = xmp_prefixed_statfs,
+ .release = xmp_release,
+ .fsync = xmp_fsync,
+#ifdef HAVE_POSIX_FALLOCATE
+ .fallocate = xmp_fallocate,
+#endif
+#ifdef HAVE_SETXATTR
+ .setxattr = xmp_setxattr,
+ .getxattr = xmp_getxattr,
+ .listxattr = xmp_listxattr,
+ .removexattr = xmp_removexattr,
+#endif
+};
+
+
+int main( int argc, char *argv[] )
+{
+ int res = 0;
+ if ( argc < 4 )
+ printf( "usage: fuse-proxy [options] mount-point proxy-dir truncate_limit write_limit\n" );
+ else
+ {
+ set_prefix_write_limit( str_2_size( argv[ argc - 1 ] ) );
+ set_prefix_truncate_limit( str_2_size( argv[ argc - 2 ] ) );
+ if ( set_prefix( argv[ argc - 3 ] ) == 0 )
+ {
+ printf( "cannot find absolute path of '%s'\n", argv[ argc - 3 ] );
+ res = 3;
+ }
+ else
+ {
+ printf( "establishing proxy '%s' -> '%s'\n", argv[ argc - 4 ], argv[ argc - 3 ] );
+ if ( prefix_write_limit > 0 )
+ printf( "with write-limit of %lu bytes\n", prefix_write_limit );
+ if ( prefix_truncate_limit > 0 )
+ printf( "with truncate-limit of %lu bytes\n", prefix_truncate_limit );
+
+ argc -= 3;
+ umask( 0 );
+ /* this function does not return! */
+ fuse_main( argc, argv, &xmp_oper, NULL );
+ }
+ }
+ return res;
+}
diff --git a/test/kfs/kdf.cpp b/test/kfs/kdf.cpp
new file mode 100644
index 0000000..e6a9aaf
--- /dev/null
+++ b/test/kfs/kdf.cpp
@@ -0,0 +1,303 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+* Tests for KDirectoryGetDiskFreeSpace.
+*/
+
+
+#include <kapp/args.h> /* ArgsMakeAndHandle */
+
+#include <kfs/directory.h> /* KDirectoryGetDiskFreeSpace */
+
+#include <klib/out.h> /* OUTMSG */
+#include <klib/printf.h> /* string_printf */
+
+#include <strtol.h> /* strtou64 */
+
+#include <ktst/unit_test.hpp> // TEST_SUITE
+
+#include <cmath> // ceil
+#include <cstdio> // popen
+
+
+static rc_t argsHandler ( int argc, char * argv [] );
+TEST_SUITE_WITH_ARGS_HANDLER ( DuSuite, argsHandler )
+static Args * args = NULL;
+static const char * s_path = NULL;
+
+
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false)
+
+
+using std::cerr;
+
+
+struct FIXTURE {
+ const char * _path;
+ const KDirectory * _dir;
+
+ FIXTURE ( const char * aPath = NULL )
+ : _path ( aPath ? aPath : s_path )
+ , _dir ( NULL )
+ {
+ if ( _path == NULL )
+ return;
+
+ if ( _path [ 0 ] == '~' && _path [ 1 ] == '\0' )
+ _path = getenv ( "HOME" );
+
+ if ( _path == NULL )
+ return;
+
+TEST_MESSAGE ( "FIXTURE: " << _path );
+
+ KDirectory * native = NULL;
+ rc_t rc = KDirectoryNativeDir ( & native );
+TEST_MESSAGE ( "KDirectoryNativeDir = " << rc );
+ if ( rc != 0 )
+ throw rc;
+
+ rc = KDirectoryOpenDirRead ( native, & _dir, false, _path );
+TEST_MESSAGE ( "KDirectoryOpenDirRead = " << rc );
+ if ( rc != 0 )
+ throw rc;
+
+ rc = KDirectoryRelease ( native );
+TEST_MESSAGE ( "KDirectoryRelease = " << rc );
+ if ( rc != 0 )
+ throw rc;
+TEST_MESSAGE ( "exiting FIXTURE()" );
+ }
+
+ ~FIXTURE ( void ) {
+ rc_t rc = 0;
+ RELEASE ( KDirectory, _dir );
+ }
+};
+
+
+/* The following 2 test cases ( Mac and Linux )
+ * generate output somewhat similar to df <dir>
+ * To generate the output run "kdf -app_args=<dir>"
+ * and visually compare it with the output of "df".
+ *
+ * The next case runs "df ~", parses its output
+ * and compare it with KDirectoryGetDiskFreeSpace results.
+ */
+
+#if MAC
+
+FIXTURE_TEST_CASE ( Mac, FIXTURE ) {
+ uint64_t free_bytes_available = 0;
+ uint64_t total_number_of_bytes = 0;
+
+ REQUIRE_RC_FAIL ( KDirectoryGetDiskFreeSpace ( NULL,
+ & free_bytes_available, & total_number_of_bytes ) );
+
+ if ( _path != NULL ) {
+ REQUIRE_RC ( KDirectoryGetDiskFreeSpace ( _dir,
+ & free_bytes_available, & total_number_of_bytes ) );
+
+ OUTMSG ( ( "Filesystem 512-blocks Used Available Capacity "
+ "iused ifree %%iused Mounted on" ) );
+
+ uint64_t used = total_number_of_bytes - free_bytes_available;
+ double percent = 100. * used / total_number_of_bytes ;
+ OUTMSG ( ( "\n %d %d %d%%",
+ used / 512, free_bytes_available / 512,
+ static_cast < int > ( ceil ( percent ) ) ) ) ;
+ }
+}
+
+#else
+
+FIXTURE_TEST_CASE ( Linux, FIXTURE ) {
+ uint64_t free_bytes_available = 0;
+ uint64_t total_number_of_bytes = 0;
+
+ REQUIRE_RC_FAIL ( KDirectoryGetDiskFreeSpace ( NULL,
+ & free_bytes_available, & total_number_of_bytes ) );
+
+ if ( _path != NULL ) {
+ REQUIRE_RC ( KDirectoryGetDiskFreeSpace ( _dir,
+ & free_bytes_available, & total_number_of_bytes ) );
+
+ OUTMSG ( (
+ "Filesystem 1K-blocks Used Available Use%% Mounted on"
+ ) );
+
+ uint64_t used = total_number_of_bytes - free_bytes_available;
+ double percent = 100. * used / total_number_of_bytes ;
+ OUTMSG ( ( "\n %d %d %d%% %s",
+ used / 1024, free_bytes_available / 1024,
+ static_cast < int > ( ceil ( percent ) ), _path ) ) ;
+ }
+}
+
+#endif
+
+struct C {
+ static bool df ( const char * command, uint64_t & free_bytes_available,
+ uint64_t & total_number_of_byte )
+ {
+ FILE * fp = popen ( command, "r" );
+ if ( fp == NULL )
+ return false;
+
+ char line [ 2 ] [ 1035 ];
+ char * crnt = line [ 0 ];
+ char * prev = line [ 1 ];
+ while ( fgets ( crnt, sizeof line [ 0 ] - 1, fp ) != NULL ) {
+ char * tmp = crnt;
+ crnt = prev;
+ prev = tmp;
+ }
+
+ pclose ( fp );
+
+ assert ( prev );
+
+ crnt = prev;
+ for ( int i = 0, in = true; * crnt != '\0'; ++ crnt ) {
+ if ( * crnt == ' ' ) {
+ if ( in ) {
+ in = false;
+ ++ i;
+ }
+ }
+ else
+ if ( ! in ) {
+ in = true;
+ if ( i == 1 || i == 3 ) {
+ uint64_t r = strtou64 ( crnt, & crnt, 10 );
+ switch ( i ) {
+ case 1:
+ total_number_of_byte = r;
+ break;
+ case 3:
+ free_bytes_available = r;
+ return true;
+ }
+ -- crnt;
+ }
+ }
+ }
+ return false;
+ }
+};
+
+TEST_CASE ( testKDirectoryGetDiskFreeSpace ) {
+ FIXTURE fixture ( "~" );
+
+ char command [ 256 ] = "";
+ TEST_MESSAGE ( fixture . _path );
+ REQUIRE_RC ( string_printf
+ ( command, sizeof command, NULL, "df -k %s", fixture . _path ) );
+
+ uint64_t total_number_of_bytes = 0;
+ uint64_t free_bytes_available = 0;
+
+ uint64_t blocks = 0;
+ uint64_t available = 0;
+
+ bool started = false;
+ int64_t min = 9876543210;
+ int i = 0;
+ for ( i = 1; i < 99; ++i ) {
+ REQUIRE_RC ( KDirectoryGetDiskFreeSpace ( fixture . _dir,
+ & free_bytes_available, & total_number_of_bytes ) );
+
+ if ( started ) {
+ if ( available == free_bytes_available / 1024 ) {
+ cerr << "DONE in " << i << ".5 iterations";
+ return;
+ }
+ }
+
+ REQUIRE ( C::df ( command, available, blocks ) );
+
+ if ( ! started ) {
+ REQUIRE_EQ ( blocks, total_number_of_bytes / 1024 );
+ started = true;
+ }
+
+ if ( available == free_bytes_available / 1024 ) {
+ cerr << "DONE in " << i << " iterations";
+ return;
+ }
+ else {
+ if ( abs ( ( int64_t ) available - ( int64_t ) free_bytes_available / 1024 ) < min )
+ min = abs ( ( int64_t ) available - ( int64_t ) free_bytes_available / 1024 );
+ cerr << i << " KDirectoryGetDiskFreeSpace="
+ << free_bytes_available / 1024
+ << " Available=" << available << " ( "
+ << abs ( ( int64_t ) free_bytes_available / 1024 - ( int64_t ) available ) << " )\n";
+ }
+ }
+
+ if ( min < 99 ) {
+ cerr << "DONE in " << i
+ << " iterations: ( available - ( free_bytes_available / 1024 ) = "
+ << min << ")\n";
+ return;
+ }
+
+ REPORT_ERROR ( "Cannot match KDirectoryGetDiskFreeSpace and df results" );
+}
+
+static rc_t argsHandler ( int argc, char * argv [] ) {
+ uint32_t params = 0;
+
+ rc_t rc = ArgsMakeAndHandle ( & args, argc, argv, 0, NULL, 0 );
+
+ if ( rc == 0 )
+ rc = ArgsParamCount ( args, & params );
+
+ if ( rc == 0 && params > 0 )
+ rc = ArgsParamValue ( args, 0,
+ reinterpret_cast < const void ** > ( & s_path ) );
+
+ return rc;
+}
+
+rc_t CC UsageSummary (const char * progname) { return 0; }
+rc_t CC Usage ( const Args * args ) { return 0; }
+const char UsageDefaultName [] = "kdf";
+
+extern "C" {
+ ver_t CC KAppVersion ( void ) { return 0; }
+
+ rc_t KMain ( int argc, char * argv [] ) {
+ncbi::NK::TestEnv::SetVerbosity(ncbi::NK::LogLevel::e_all);
+ rc_t rc = DuSuite ( argc, argv );
+
+ s_path = NULL;
+
+ ArgsWhack ( args );
+ args = NULL;
+
+ return rc;
+ }
+}
diff --git a/test/kfs/run-cachetee-out-of-space-test.sh b/test/kfs/run-cachetee-out-of-space-test.sh
new file mode 100755
index 0000000..cc7f352
--- /dev/null
+++ b/test/kfs/run-cachetee-out-of-space-test.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+RUN_DIR="$1"
+BIN_DIR="$2"
+
+echo "Out of space test #1 ... there is no space to create the cache-tee-file from the beginning"
+#=====================================================================================================
+
+#try to get rid of fuse-mount ( in case the test was terminated, before this script was able
+#to unmount the fuse-mount-point...
+fusermount -u $RUN_DIR/mount_point
+
+rm -rf $RUN_DIR/mount_point $RUN_DIR/proxy_point
+
+mkdir $RUN_DIR/mount_point
+rc=$?; if [[ $rc -ne 0 ]]; then echo "error creating mount-point"; exit $rc; fi
+
+mkdir $RUN_DIR/proxy_point
+rc=$?; if [[ $rc -ne 0 ]]; then echo "error creating proxy-point"; exit $rc; fi
+
+TRUNCATE_LIMIT="10k"
+WRITE_LIMIT="0"
+$BIN_DIR/fuse-proxy $RUN_DIR/mount_point $RUN_DIR/proxy_point $TRUNCATE_LIMIT $WRITE_LIMIT
+rc=$?; if [[ $rc -ne 0 ]]; then echo "error starting fuse-proxy"; exit $rc; fi
+
+$BIN_DIR/test-cachetee-out-of-space
+rc_test=$?
+
+fusermount -u $RUN_DIR/mount_point
+rc=$?; if [[ $rc -ne 0 ]]; then echo "error stopping fuse-proxy"; exit $rc; fi
+
+rm -rf $RUN_DIR/mount_point $RUN_DIR/proxy_point
+
+if [[ $rc_test -ne 0 ]]; then echo "error in test-cachetee-out-of-space binary"; exit $rc_test; fi
+
+
+echo "Out of space test #2 ... there is no space while using the cache-tee-file"
+#=====================================================================================================
+#try to get rid of fuse-mount ( in case the test was terminated, before this script was able
+#to unmount the fuse-mount-point...
+fusermount -u $RUN_DIR/mount_point
+
+rm -rf $RUN_DIR/mount_point $RUN_DIR/proxy_point
+
+mkdir $RUN_DIR/mount_point
+rc=$?; if [[ $rc -ne 0 ]]; then echo "error creating mount-point"; exit $rc; fi
+
+mkdir $RUN_DIR/proxy_point
+rc=$?; if [[ $rc -ne 0 ]]; then echo "error creating proxy-point"; exit $rc; fi
+
+TRUNCATE_LIMIT="0"
+WRITE_LIMIT="10k"
+$BIN_DIR/fuse-proxy $RUN_DIR/mount_point $RUN_DIR/proxy_point $TRUNCATE_LIMIT $WRITE_LIMIT
+rc=$?; if [[ $rc -ne 0 ]]; then echo "error starting fuse-proxy"; exit $rc; fi
+
+$BIN_DIR/test-cachetee-out-of-space
+rc_test=$?
+
+fusermount -u $RUN_DIR/mount_point
+rc=$?; if [[ $rc -ne 0 ]]; then echo "error stopping fuse-proxy"; exit $rc; fi
+
+rm -rf $RUN_DIR/mount_point $RUN_DIR/proxy_point
+
+if [[ $rc_test -ne 0 ]]; then echo "error in test-cachetee-out-of-space binary"; exit $rc_test; fi
diff --git a/test/klib/Makefile b/test/klib/Makefile
index a3be7f1..6fd3dec 100644
--- a/test/klib/Makefile
+++ b/test/klib/Makefile
@@ -36,6 +36,7 @@ TEST_TOOLS = \
test-log \
test-out \
test-SraReleaseVersion \
+ test-time \
test-klib \
test-vnamelist
@@ -143,6 +144,18 @@ valgrind_sra_release_version:
#-------------------------------------------------------------------------------
+# test-time
+#
+TEST_TIME_SRC = \
+ test-time
+
+TEST_TIME_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_TIME_SRC))
+
+$(TEST_BINDIR)/test-time: $(TEST_TIME_OBJ)
+ $(LP) --exe -o $@ $^ $(TEST_SRA_RELEASE_VERSION_LIB)
+
+#-------------------------------------------------------------------------------
# test-out
#
TEST_OUT_SRC = \
diff --git a/test/klib/test-log.cpp b/test/klib/test-log.cpp
index 397439e..2f85e99 100644
--- a/test/klib/test-log.cpp
+++ b/test/klib/test-log.cpp
@@ -293,6 +293,42 @@ TEST_CASE(UninitailizedCrachTest) {
PLOGMSG(klogWarn, (klogWarn, "Now you see $(a)", "a=%s", "a warning")));
}
+TEST_CASE ( rcBufferrcInsufficientAfterprep_v_args ) {
+ REQUIRE_RC ( KWrtInit ( "fastq-dump", 0x02070000 ) );
+ REQUIRE_RC ( KLogLibHandlerSetStdErr () );
+
+ rc_t rc = SILENT_RC ( rcNS, rcFile, rcOpening, rcFunction, rcUnsupported );
+
+ size_t s ( 1969 );
+ string u ( s + 1, '<' );
+ u += "0123456789ABCDEFGHIJKLMNOPQRST";
+ string v ( s, '>' );
+ v += "56789ABCDEFGHIJKLMNOPRSTUVWXYZ";
+
+ REQUIRE_RC ( pLogLibErr ( klogErr, rc, "This message "
+ "used to produce a log failure after error prep_v_args call: "
+ "error with http open '$(U) * $(V)'",
+ "U=%s,V=%s", u.c_str (), v.c_str () ) );
+}
+
+TEST_CASE ( rcBufferrcInsufficientInprep_v_argsstring_vprintf ) {
+ REQUIRE_RC ( KWrtInit ( "fastq-dump", 0x02070000 ) );
+ REQUIRE_RC ( KLogLibHandlerSetStdErr () );
+
+ rc_t rc = SILENT_RC ( rcNS, rcFile, rcOpening, rcFunction, rcUnsupported );
+
+ size_t s ( 2025 );
+ string u ( s + 1, '<' );
+ u += "0123456789ABCDEFGHIJ";
+ string v ( s, '>' );
+ v += "FGHIJKLMNOPRSTUVWXYZ";
+
+ REQUIRE_RC ( pLogLibErr ( klogErr, rc, "This message "
+ "used to produce a log failure inside of prep_v_args call: "
+ "error with http open '$(U) * $(V)'",
+ "U=%s,V=%s", u.c_str (), v.c_str () ) );
+}
+
//TODO:
// KLogFmtFlagsSet
// KLogLibFmtFlagsSet
diff --git a/test/klib/test-time.cpp b/test/klib/test-time.cpp
new file mode 100644
index 0000000..55ea266
--- /dev/null
+++ b/test/klib/test-time.cpp
@@ -0,0 +1,87 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <klib/time.h> /* KTime */
+#include <ktst/unit_test.hpp> // TEST_SUITE
+
+using std::string;
+
+TEST_SUITE ( TestTimeSuite );
+
+TEST_CASE ( test ) {
+
+////////////////////////////////////////////////////////////////////////////////
+
+ TEST_MESSAGE (
+ "Checking roundtrip conversion Iso8601 string -> KTime -> Iso8601 string"
+ );
+
+ const string date1 ( "2015-04-07T21:54:15Z" );
+
+ KTime kt1;
+ const KTime * t
+ = KTimeFromIso8601 ( & kt1, date1 . c_str (), date1 . size () );
+ REQUIRE ( t );
+
+ KTime_t ts = KTimeMakeTime ( & kt1 );
+
+ char date2 [ 64 ];
+ size_t sz = KTimeIso8601 ( ts, date2, sizeof date2 );
+
+ REQUIRE ( sz );
+ REQUIRE_EQ ( sz, date1 . size () );
+ REQUIRE_EQ ( date2 [ sz ], '\0' );
+ REQUIRE_EQ ( string ( date2 ), date1 );
+
+////////////////////////////////////////////////////////////////////////////////
+
+ TEST_MESSAGE ( "Checking roundtrip conversion 'KTime kt1' -> "
+ "'KTime_t ts = KTimeMakeTime(kt1)' -> 'KTime kt2 = KTimeGlobal(ts)'" );
+
+ KTime kt2;
+ const KTime * ktp = KTimeGlobal ( & kt2, ts );
+
+ REQUIRE ( ktp );
+ REQUIRE ( kt1 . year==kt2 . year && kt1 . month == kt2 . month );
+ REQUIRE ( kt1 . day == kt2.day );
+ if ( kt1 . weekday )
+ REQUIRE_EQ ( kt1 . weekday, kt2 . weekday );
+#if !defined(__SunOS) && !defined(__sun__)
+ REQUIRE_EQ ( kt1 . tzoff, kt2 . tzoff );
+#endif
+ REQUIRE_EQ ( kt1 . hour, kt2 . hour );
+ REQUIRE ( kt1 . minute == kt2 . minute && kt1 . second == kt2 . second );
+ REQUIRE_EQ ( kt1 . dst , kt2 . dst );
+
+////////////////////////////////////////////////////////////////////////////////
+
+}
+
+extern "C" {
+ ver_t CC KAppVersion ( void ) { return 0; }
+ rc_t CC KMain ( int argc, char * argv [] )
+ { return TestTimeSuite ( argc, argv ); }
+}
diff --git a/test/kns/Makefile b/test/kns/Makefile
index 6c809f3..018525c 100644
--- a/test/kns/Makefile
+++ b/test/kns/Makefile
@@ -25,20 +25,23 @@
default: runtests
+runtests: run-test-proxy-with-scheme
+
TOP ?= $(abspath ../..)
MODULE = test/kns
TEST_TOOLS = \
- KNSManagerSingletonTest \
test-kns \
test-proxy \
test-http \
test-http-dropconn \
+ KNSManagerSingletonTest \
include $(TOP)/build/Makefile.env
EXT_TOOLS = \
- test-proxy-with-env
+ test-proxy-with-env \
+ test-proxy-with-scheme \
ALL_TOOLS = \
$(INT_TOOLS) \
@@ -56,6 +59,14 @@ $(ALL_TOOLS): makedirs
.PHONY: all std $(ALL_TOOLS)
#-------------------------------------------------------------------------------
+# all
+#
+$(TARGDIR)/all: \
+ $(addprefix $(BINDIR)/,$(ALL_TOOLS))
+
+.PHONY: $(TARGDIR)/all
+
+#-------------------------------------------------------------------------------
# std
#
$(TARGDIR)/std: \
@@ -112,7 +123,7 @@ KNSTEST_OBJ = \
$(TEST_BINDIR)/test-kns: $(KNSTEST_OBJ)
$(LP) --exe -o $@ $^ $(KNSTEST_LIB)
-kns: test-kns
+kns: test-kns
$(TEST_BINDIR)/test-kns #-l=all
#----------------------------------------------------------------
@@ -127,7 +138,21 @@ PROXYTEST_OBJ = \
$(TEST_BINDIR)/test-proxy: $(PROXYTEST_OBJ)
$(LP) --exe -o $@ $^ $(KNSTEST_LIB)
-
+
+
+#----------------------------------------------------------------
+# test-proxy-with-scheme
+#
+PROXYSCH_SRC = \
+ test-proxy-with-scheme \
+
+PROXYSCH_OBJ = \
+ $(addsuffix .$(OBJX),$(PROXYSCH_SRC))
+
+$(BINDIR)/test-proxy-with-scheme: $(PROXYSCH_OBJ)
+ $(LP) --exe -o $@ $^ $(KNSTEST_LIB)
+
+
#----------------------------------------------------------------
# test-http
#
@@ -140,9 +165,9 @@ HTTPTEST_OBJ = \
$(TEST_BINDIR)/test-http: $(HTTPTEST_OBJ)
$(LP) --exe -o $@ $^ $(KNSTEST_LIB)
-http: test-http
+http: test-http
$(TEST_BINDIR)/test-http # -l=all
-
+
#----------------------------------------------------------------
# test-http-dropconn
#
@@ -155,12 +180,12 @@ HTTP_DROPCONN_TEST_OBJ = \
$(TEST_BINDIR)/test-http-dropconn: $(HTTP_DROPCONN_TEST_OBJ)
$(LP) --exe -o $@ $^ $(KNSTEST_LIB)
-dropconn: test-http-dropconn
+dropconn: test-http-dropconn
$(TEST_BINDIR)/test-http-dropconn # -l=all
-
+
#----------------------------------------------------------------
# test-proxy-with-env
-#
+#
TOOL_SRC = \
http-client \
test-proxy-with-env \
@@ -170,3 +195,58 @@ TOOL_OBJ = \
$(BINDIR)/test-proxy-with-env: $(TOOL_OBJ)
$(LP) --exe -o $@ $^ -L$(VDB_LIBDIR) -L$(VDB_LIBDIR)/../ilib $(KNSTEST_LIB)
+
+
+#----------------------------------------------------------------
+
+
+# test-proxy-with-scheme never does direct intermet connection
+# ( /http/proxy/only = "true" )
+# It can access internet
+# just when there is a good proxy specified via configuration or environment
+
+run-test-proxy-with-scheme:
+ @ echo test-proxy-with-scheme: BEGIN ...
+
+ # no proxy specified: fail
+ $(BINDIR)/test-proxy-with-scheme
+
+ # bad proxy in configuration: fail
+ $(BINDIR)/test-proxy-with-scheme =BAD.proxy.ncbi.nlm.nih.gov
+
+ # good proxy in configuration: success
+ $(BINDIR)/test-proxy-with-scheme =webproxy.ncbi.nlm.nih.gov SUCCESS
+
+ # bad proxy in environment: fail
+ all_proxy=BAD.proxy.ncbi.nlm.nih.gov $(BINDIR)/test-proxy-with-scheme
+
+ # good proxy in environment: success
+ all_proxy=webproxy.ncbi.nlm.nih.gov \
+ $(BINDIR)/test-proxy-with-scheme SUCCESS
+
+ # good proxy with schema in environment: success
+ ALL_PROXY=http://webproxy.ncbi.nlm.nih.gov \
+ $(BINDIR)/test-proxy-with-scheme SUCCESS
+
+ # good proxy with schema and port in environment: success
+ http_proxy=http://webproxy.ncbi.nlm.nih.gov:3128 \
+ $(BINDIR)/test-proxy-with-scheme SUCCESS
+
+ # good proxy with any schema in environment: success
+ HTTP_PROXY=anySchemaWillBeIgnored://webproxy.ncbi.nlm.nih.gov \
+ $(BINDIR)/test-proxy-with-scheme SUCCESS
+
+ # bad proxy port in environment: fail
+ all_proxy=http://webproxy.ncbi.nlm.nih.gov:3 \
+ $(BINDIR)/test-proxy-with-scheme
+
+ # bad proxy specifications in environment: fail
+ all_proxy=h $(BINDIR)/test-proxy-with-scheme
+ all_proxy=: $(BINDIR)/test-proxy-with-scheme
+ all_proxy=:a $(BINDIR)/test-proxy-with-scheme
+ all_proxy=:2 $(BINDIR)/test-proxy-with-scheme
+ all_proxy=h:/w.g:3128 $(BINDIR)/test-proxy-with-scheme
+ all_proxy=http:// $(BINDIR)/test-proxy-with-scheme
+ all_proxy=http://:3128 $(BINDIR)/test-proxy-with-scheme
+
+ @ echo ... test-proxy-with-scheme: END
diff --git a/test/kns/test-proxy-with-scheme.cpp b/test/kns/test-proxy-with-scheme.cpp
new file mode 100644
index 0000000..e5b5925
--- /dev/null
+++ b/test/kns/test-proxy-with-scheme.cpp
@@ -0,0 +1,162 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+
+#include <kapp/args.h> /* Args */
+
+#include <kfg/kfg-priv.h> /* KConfigMakeEmpty */
+
+#include <klib/out.h> /* KOutMsg */
+
+#include <kns/manager.h> /* KNSManagerSetConnectionTimeouts */
+
+#include <vfs/manager.h> /* VFSManagerRelease */
+#include <vfs/manager-priv.h> /* VFSManagerMakeFromKfg */
+#include <vfs/path.h> /* VFSManagerMakeAccPath */
+#include <vfs/resolver.h> /* VResolverQuery */
+
+#include <ktst/unit_test.hpp> // TEST_SUITE
+
+#define RELEASE( type, obj ) do { rc_t rc2 = type##Release ( obj ); \
+ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while ( false )
+
+static rc_t argsHandler(int argc, char* argv[]);
+TEST_SUITE_WITH_ARGS_HANDLER(TestProxySchemeSuite, argsHandler);
+
+static bool EXPECTED_FAILURE = true;
+static char * PROXY = NULL;
+
+TEST_CASE ( test ) {
+ rc_t rc = 0;
+
+ KConfig * kfg = NULL;
+ REQUIRE_NULL ( kfg );
+ REQUIRE_RC ( KConfigMakeEmpty ( & kfg ) );
+ REQUIRE_RC ( KConfigWriteString ( kfg,
+ "/repository/remote/main/CGI/resolver-cgi",
+ "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" ) );
+
+ if ( PROXY != NULL )
+ REQUIRE_RC ( KConfigWriteString ( kfg, "/http/proxy/path", PROXY ) );
+
+ /* do not try direct http access, use proxy only: VDB-3015 */
+ REQUIRE_RC ( KConfigWriteString ( kfg, "/http/proxy/only", "true" ) );
+
+ REQUIRE_NOT_NULL ( kfg );
+
+ VFSManager * vfs = NULL;
+ REQUIRE_NULL ( vfs );
+ REQUIRE_RC ( VFSManagerMakeFromKfg ( & vfs, kfg ) );
+ REQUIRE_NOT_NULL ( vfs );
+
+ KNSManager * kns = NULL;
+ REQUIRE_RC ( VFSManagerGetKNSMgr ( vfs, & kns ) );
+ // do not do long retries when calling proxy on bad port
+ REQUIRE_RC ( KNSManagerSetConnectionTimeouts ( kns, 1, 0, 0 ) );
+ RELEASE ( KNSManager, kns );
+
+ VResolver * resolver = NULL;
+ REQUIRE_RC ( VFSManagerMakeResolver ( vfs, & resolver, kfg ) );
+ REQUIRE_NOT_NULL ( resolver );
+
+ VPath * query = NULL;
+ REQUIRE_NULL ( query );
+ REQUIRE_RC ( VFSManagerMakeAccPath ( vfs, & query, "SRR000001" ) );
+ REQUIRE_NOT_NULL ( query );
+
+ const VPath * rmt = NULL;
+ REQUIRE_NULL ( rmt );
+ if ( EXPECTED_FAILURE )
+ REQUIRE_RC_FAIL ( VResolverQuery ( resolver, 0, query, 0, & rmt, 0 ) );
+ else
+ REQUIRE_RC ( VResolverQuery ( resolver, 0, query, 0, & rmt, 0 ) );
+ RELEASE ( VPath , rmt );
+
+ RELEASE ( VPath , query );
+
+ RELEASE ( VResolver , resolver );
+
+ RELEASE ( VFSManager, vfs );
+
+ RELEASE ( KConfig , kfg );
+
+ REQUIRE_RC ( rc );
+}
+
+rc_t CC Usage ( const Args * args ) { return 0; }
+const char UsageDefaultName [] = "test-proxy-with-scheme";
+
+rc_t CC UsageSummary ( const char * prog_name ) {
+ return KOutMsg (
+"Usage:"
+""
+" test-proxy-with-scheme"
+" - expect failure"
+""
+" test-proxy-with-scheme =<proxy-spec>"
+" - set <proxy spec> in configuration;"
+" - expect failure"
+" Example:"
+" test-proxy-with-scheme =bad.proxy"
+ );
+}
+
+static rc_t argsHandler ( int argc, char * argv [] ) {
+ rc_t rc = ArgsMakeAndHandle ( NULL, argc, argv, 0, NULL, 0 );
+ if ( rc != 0 )
+ return rc;
+
+ if ( argc > 1 )
+ if ( argv [ 1 ] [ 0 ] == '=' )
+ PROXY = strdup ( & argv [ 1 ] [ 1 ] );
+
+ switch ( argc ) {
+ case 1:
+ EXPECTED_FAILURE = true;
+ break;
+ case 2:
+ EXPECTED_FAILURE = PROXY != NULL;
+ break;
+ default:
+ EXPECTED_FAILURE = false;
+ break;
+ }
+
+ return rc;
+}
+
+extern "C" {
+ ver_t CC KAppVersion ( void ) { return 0; }
+
+ rc_t CC KMain ( int argc, char * argv [] ) {
+ rc_t rc = TestProxySchemeSuite ( argc, argv );
+
+ free ( PROXY );
+ PROXY = NULL;
+
+ return rc;
+ }
+}
diff --git a/test/kns/test-proxy.cpp b/test/kns/test-proxy.cpp
index 40a51ad..334250c 100644
--- a/test/kns/test-proxy.cpp
+++ b/test/kns/test-proxy.cpp
@@ -252,6 +252,22 @@ TEST_CASE ( TEST_PROXY_ENV_commaErr ) {
}
#endif
+TEST_CASE ( TEST_PROXY_ENV_envWithSchema ) {
+ TestRunner ( this, "env-with-schema" );
+}
+
+TEST_CASE ( TEST_PROXY_ENV_envWithSchemaAndPort ) {
+ TestRunner ( this, "env-with-schema-and-port" );
+}
+
+TEST_CASE ( TEST_PROXY_ENV_kfgWithSchema ) {
+ TestRunner ( this, "kfg-with-schema" );
+}
+
+TEST_CASE ( TEST_PROXY_ENV_kfgWithSchemaAndPort ) {
+ TestRunner ( this, "kfg-with-schema-and-port" );
+}
+
extern "C" {
ver_t CC KAppVersion ( void ) { return 0; }
rc_t CC KMain ( int argc, char * argv [] ) {
diff --git a/test/kns/test-proxy/env-with-schema-and-port/environment b/test/kns/test-proxy/env-with-schema-and-port/environment
new file mode 100644
index 0000000..75485d8
--- /dev/null
+++ b/test/kns/test-proxy/env-with-schema-and-port/environment
@@ -0,0 +1 @@
+HTTP_PROXY=anySchemaIsIgnored://http.proxy:1234
diff --git a/test/kns/test-proxy/env-with-schema-and-port/expected b/test/kns/test-proxy/env-with-schema-and-port/expected
new file mode 100644
index 0000000..33af3c4
--- /dev/null
+++ b/test/kns/test-proxy/env-with-schema-and-port/expected
@@ -0,0 +1 @@
+http.proxy 1234
diff --git a/test/kns/test-proxy/env-with-schema/environment b/test/kns/test-proxy/env-with-schema/environment
new file mode 100644
index 0000000..dff67df
--- /dev/null
+++ b/test/kns/test-proxy/env-with-schema/environment
@@ -0,0 +1 @@
+http_proxy=http://no-port.http.proxy
diff --git a/test/kns/test-proxy/env-with-schema/expected b/test/kns/test-proxy/env-with-schema/expected
new file mode 100644
index 0000000..e2944d8
--- /dev/null
+++ b/test/kns/test-proxy/env-with-schema/expected
@@ -0,0 +1 @@
+no-port.http.proxy 0
diff --git a/test/kns/test-proxy/kfg-with-schema-and-port/config b/test/kns/test-proxy/kfg-with-schema-and-port/config
new file mode 100644
index 0000000..a6be4b6
--- /dev/null
+++ b/test/kns/test-proxy/kfg-with-schema-and-port/config
@@ -0,0 +1 @@
+/http/proxy/path anySchemaIsIgnored://http.proxy:1234
diff --git a/test/kns/test-proxy/kfg-with-schema-and-port/expected b/test/kns/test-proxy/kfg-with-schema-and-port/expected
new file mode 100644
index 0000000..33af3c4
--- /dev/null
+++ b/test/kns/test-proxy/kfg-with-schema-and-port/expected
@@ -0,0 +1 @@
+http.proxy 1234
diff --git a/test/kns/test-proxy/kfg-with-schema/config b/test/kns/test-proxy/kfg-with-schema/config
new file mode 100644
index 0000000..30c8514
--- /dev/null
+++ b/test/kns/test-proxy/kfg-with-schema/config
@@ -0,0 +1 @@
+/http/proxy/path http://no-port.http.proxy
diff --git a/test/kns/test-proxy/kfg-with-schema/expected b/test/kns/test-proxy/kfg-with-schema/expected
new file mode 100644
index 0000000..e2944d8
--- /dev/null
+++ b/test/kns/test-proxy/kfg-with-schema/expected
@@ -0,0 +1 @@
+no-port.http.proxy 0
diff --git a/test/krypto/Makefile b/test/krypto/Makefile
index 67fb045..9786c71 100644
--- a/test/krypto/Makefile
+++ b/test/krypto/Makefile
@@ -71,6 +71,14 @@ $(ALL_TOOLS): makedirs
.PHONY: all std $(ALL_TOOLS)
#-------------------------------------------------------------------------------
+# all
+#
+$(TARGDIR)/all: \
+ $(addprefix $(BINDIR)/,$(ALL_TOOLS))
+
+.PHONY: $(TARGDIR)/all
+
+#-------------------------------------------------------------------------------
# std
#
$(TARGDIR)/std: \
@@ -122,7 +130,7 @@ $(TEST_BINDIR)/test-aes-ciphers: $(TEST_AES_CIPHERS_OBJ)
# test cipher speed
# run a null cipher and the aes cipher in all implmentations supported on the
# test machine.
-# Comparing specific two versions will test possible speed gains from verious
+# Comparing specific two versions will test possible speed gains from verious
# optimizations. Comparing the null cipher (just a copy) versions against each
# other offer insight into whether vector and vector register optimiztions are
# beneficial and by how much
diff --git a/test/ktst/ktsttest.cpp b/test/ktst/ktsttest.cpp
index fdb9240..a8669c5 100644
--- a/test/ktst/ktsttest.cpp
+++ b/test/ktst/ktsttest.cpp
@@ -150,6 +150,58 @@ TEST_CASE(ArgHandlerCalled)
REQUIRE(argHandlerCalled);
}
+
+struct Shared : SharedTest {
+ Shared ( TestCase * dad, bool ok = true ) : SharedTest ( dad, "Shared" ) {
+ CHECK ( ok );
+ }
+};
+
+TEST_CASE ( SharedSucceed ) {
+ REQUIRE_EQ ( this -> GetErrorCounter (), 0 );
+ Shared test ( this );
+ REQUIRE_EQ ( this -> GetErrorCounter (), 0 );
+}
+
+TEST_CASE ( SharedFailed ) {
+ REQUIRE_EQ ( GetErrorCounter (), 0 ); // no error when starting
+ Shared test ( this, false ); // make shared SharedTest
+
+ // ErrorCounter is adjusted when SharedTest goes out of scope
+ REQUIRE_EQ ( GetErrorCounter (), 0 );
+
+ // compensate SharedTest failure - make this test case succeed
+ ErrorCounterAdd ( -1 );
+}
+
+TEST_CASE ( SharedFailedAndDetected ) {
+ REQUIRE_EQ ( GetErrorCounter (), 0 ); // no error when starting
+
+ {
+ Shared test ( this, false ); // make SharedTest fail
+
+ // ErrorCounter is adjusted when SharedTest goes out of scope
+ REQUIRE_EQ ( GetErrorCounter (), 0 );
+ }
+ REQUIRE_EQ ( GetErrorCounter (), 1 );
+
+ // compensate SharedTest failure - make this test case succeed
+ ErrorCounterAdd ( -1 );
+}
+
+TEST_CASE ( SharedSucceedInBlock ) {
+ REQUIRE_EQ ( GetErrorCounter (), 0 ); // no error when starting
+
+ {
+ Shared test ( this ); // make SharedTest succeed
+
+ // ErrorCounter is adjusted when SharedTest goes out of scope
+ REQUIRE_EQ ( GetErrorCounter (), 0 );
+ }
+ REQUIRE_EQ ( GetErrorCounter (), 0 );
+}
+
+
//TODO: test FIXTURE_TEST_CASE, PROCESS_FIXTURE_TEST_CASE
//TODO: test GET_GLOBAL_FIXTURE
//TODO: test REQUIRE_THROW, THROW_ON_RC
diff --git a/test/ngs-java/ngs_test_CSRA1.java b/test/ngs-java/ngs_test_CSRA1.java
index e38d055..b4f04c2 100644
--- a/test/ngs-java/ngs_test_CSRA1.java
+++ b/test/ngs-java/ngs_test_CSRA1.java
@@ -39,9 +39,9 @@ import ngs.ErrorMsg;
import gov.nih.nlm.ncbi.ngs.NGS;
-// The purpose of this suite is to verify integration of Java/JNI/C code,
-// for which running through just one type of archive is enough.
-// Thus these tests are not replicated for SRA and SRADB
+// The purpose of this suite is to verify integration of Java/JNI/C code,
+// for which running through just one type of archive is enough.
+// Thus these tests are not replicated for SRA and SRADB
// archives, unlike in the C-level test suites
public class ngs_test_CSRA1 {
@@ -51,13 +51,13 @@ public class ngs_test_CSRA1 {
String WithGroups = "SRR822962";
String WithCircularRef = "SRR1769246";
String SingleFragmentPerSpot = "SRR2096940";
-
+
@Test
public void open_success() throws ngs.ErrorMsg
{
ngs.ReadCollection run = NGS . openReadCollection ( PrimaryOnly );
}
-
+
@Test
public void open_fail()
{
@@ -66,33 +66,33 @@ public class ngs_test_CSRA1 {
ngs.ReadCollection run = NGS . openReadCollection ( "SRRsomejunk" );
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
-
+
@Test
public void ReadCollection_getName() throws ngs.ErrorMsg
{
assertEquals ( PrimaryOnly, NGS . openReadCollection ( PrimaryOnly ) . getName () );
}
-
+
@Test
public void ReadCollection_getReadGroup() throws ngs.ErrorMsg
{
ngs.ReadGroup gr = NGS . openReadCollection ( PrimaryOnly ) . getReadGroup ( "C1ELY.6" );
}
-
+
@Test
public void ReadCollection_getReadGroups() throws ngs.ErrorMsg
{
ngs.ReadGroupIterator it = NGS . openReadCollection ( PrimaryOnly ) . getReadGroups ();
}
-
+
@Test
public void ReadCollection_getReferences() throws ngs.ErrorMsg
{
ngs.ReferenceIterator it = NGS . openReadCollection ( PrimaryOnly ) . getReferences ();
}
-
+
@Test
public void ReadCollection_getReference() throws ngs.ErrorMsg
{
@@ -105,13 +105,13 @@ public class ngs_test_CSRA1 {
assert ( NGS . openReadCollection ( PrimaryOnly ) . hasReference ( "supercont2.1" ) );
assert ( ! NGS . openReadCollection ( PrimaryOnly ) . hasReference ( "non-existent acc" ) );
}
-
+
@Test
public void ReadCollection_getAlignment() throws ngs.ErrorMsg
{
ngs.Alignment al = NGS . openReadCollection ( PrimaryOnly ) . getAlignment( PrimaryOnly + ".PA.1" );
}
-
+
@Test
public void ReadCollection_getAlignments_Primary() throws ngs.ErrorMsg
{
@@ -148,7 +148,7 @@ public class ngs_test_CSRA1 {
{
assertEquals ( 3987701, NGS . openReadCollection ( PrimaryOnly ) . getAlignmentCount ( Alignment . all ) );
}
-
+
@Test
public void ReadCollection_getAlignmentCount_WithSecondary() throws ngs.ErrorMsg
{
@@ -169,7 +169,7 @@ public class ngs_test_CSRA1 {
{
assertEquals ( 178, NGS . openReadCollection ( WithSecondary ) . getAlignmentCount ( Alignment . all ) );
}
-
+
@Test
public void ReadCollection_getAlignmentRange() throws ngs.ErrorMsg
{ // straddling primary and secondary alignments
@@ -177,44 +177,44 @@ public class ngs_test_CSRA1 {
assertTrue ( alIt . nextAlignment () );
assertEquals ( WithSecondary + ".PA.166", alIt . getAlignmentId () );
}
-
+
@Test
public void ReadCollection_getRead () throws ngs.ErrorMsg
- {
+ {
ngs . Read read = NGS . openReadCollection ( PrimaryOnly ) . getRead ( PrimaryOnly + ".R.1" );
assertEquals ( PrimaryOnly + ".R.1", read . getReadId () );
}
@Test
public void ReadCollection_getReads () throws ngs.ErrorMsg
- {
+ {
ngs . ReadIterator readIt = NGS . openReadCollection ( PrimaryOnly ) . getReads( ngs . Read . all );
assertTrue ( readIt . nextRead () );
assertEquals ( PrimaryOnly + ".R.1", readIt . getReadId () );
}
-
+
@Test
public void ReadCollection_getReadCount () throws ngs.ErrorMsg
- {
+ {
assertEquals ( 2280633, NGS . openReadCollection ( PrimaryOnly ) . getReadCount () );
}
-
+
@Test
public void ReadCollection_getReadRange () throws ngs.ErrorMsg
- {
+ {
ngs . ReadIterator readIt = NGS . openReadCollection ( PrimaryOnly ) . getReadRange ( 2, 3 );
assertTrue ( readIt . nextRead () );
assertEquals ( PrimaryOnly + ".R.2", readIt . getReadId () );
}
-// Read
+// Read
ngs.Read getRead ( String id ) throws ngs.ErrorMsg
{
ngs.ReadCollection run = NGS . openReadCollection ( PrimaryOnly );
return run . getRead(id);
}
-
+
@Test
public void Read_getReadCategory_full() throws ngs.ErrorMsg
{
@@ -225,13 +225,13 @@ public class ngs_test_CSRA1 {
{
assertEquals( ngs . Read . partiallyAligned, getRead( PrimaryOnly + ".R.3" ) . getReadCategory() );
}
-
+
@Test
public void Read_getNumFragments() throws ngs.ErrorMsg
{
assertEquals( 2, getRead( PrimaryOnly + ".R.1" ) . getNumFragments() );
}
-
+
@Test
public void Read_fragmentIsAligned_partial() throws ngs.ErrorMsg
{
@@ -244,12 +244,12 @@ public class ngs_test_CSRA1 {
@Test
public void FragmentIterator_ThrowsBeforeNext() throws ngs.ErrorMsg
{
- try
+ try
{
getRead( PrimaryOnly + ".R.1" ) . getFragmentId();
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
@Test
public void FragmentIterator_Next() throws ngs.ErrorMsg
@@ -266,7 +266,7 @@ public class ngs_test_CSRA1 {
assertTrue ( read . nextFragment () );
assertEquals( PrimaryOnly + ".FR0.1", read . getFragmentId () );
}
-
+
@Test
public void getFragmentBases() throws ngs.ErrorMsg
{
@@ -275,7 +275,7 @@ public class ngs_test_CSRA1 {
assertTrue ( read . nextFragment () );
assertEquals( "GGTA", read . getFragmentBases ( 2, 4 ) );
}
-
+
@Test
public void getFragmentQualities() throws ngs.ErrorMsg
{
@@ -285,9 +285,9 @@ public class ngs_test_CSRA1 {
assertEquals( "@DDA", read . getFragmentQualities ( 2, 4 ) );
}
-
+
// Alignment
-
+
ngs.Alignment getAlignment ( String id ) throws ngs.ErrorMsg
{
ngs.ReadCollection run = NGS . openReadCollection ( PrimaryOnly );
@@ -304,7 +304,7 @@ public class ngs_test_CSRA1 {
{
assertEquals( PrimaryOnly + ".PA.1", getAlignment ( PrimaryOnly + ".PA.1" ).getAlignmentId() );
}
-
+
@Test
public void Alignment_getReferenceSpec() throws ngs.ErrorMsg
{
@@ -316,20 +316,20 @@ public class ngs_test_CSRA1 {
{
assertEquals( 60, getAlignment ( PrimaryOnly + ".PA.1" ).getMappingQuality() );
}
-
+
@Test
public void Alignment_getReferenceBases() throws ngs.ErrorMsg
{
- assertEquals( "ACTCGACATTCTGTCTTCGACCTATCTTTCTCCTCTCCCAGTCATCGCCCAGTAGAATTACCAGGCAATGAACCACGGCCTTTCATCCCAACGGCACAGCA",
+ assertEquals( "ACTCGACATTCTGTCTTCGACCTATCTTTCTCCTCTCCCAGTCATCGCCCAGTAGAATTACCAGGCAATGAACCACGGCCTTTCATCCCAACGGCACAGCA",
getAlignment ( PrimaryOnly + ".PA.1" ).getReferenceBases() );
}
-
+
@Test
public void Alignment_getReadGroup() throws ngs.ErrorMsg
{
assertEquals( "C1ELY.6", getAlignment ( PrimaryOnly + ".PA.1" ).getReadGroup() );
}
-
+
@Test
public void Alignment_getReadId() throws ngs.ErrorMsg
{
@@ -341,60 +341,60 @@ public class ngs_test_CSRA1 {
{
assertEquals( PrimaryOnly + ".FA0.1", getAlignment ( PrimaryOnly + ".PA.1" ) . getFragmentId () );
}
-
+
@Test
public void Alignment_getFragmentBases_Raw() throws ngs.ErrorMsg
{
- assertEquals( "TGGATGCTCTGGAAAATCTGAAAAGTGGTGTTTGTAAGGTTTGCTGGCTGCCCATATACCACATGGATGATGGGGCTTTCCATTTTAATGTTGAAGGAGGA",
+ assertEquals( "TGGATGCTCTGGAAAATCTGAAAAGTGGTGTTTGTAAGGTTTGCTGGCTGCCCATATACCACATGGATGATGGGGCTTTCCATTTTAATGTTGAAGGAGGA",
getAlignment ( PrimaryOnly + ".PA.4" ).getFragmentBases () );
}
-
+
@Test
public void Alignment_getFragmentQualities_Raw() throws ngs.ErrorMsg
{
- assertEquals( "######AA>55;5(;63;;3@;A9??;6..73CDCIDA>DCB>@B=;@B?;;ADAB<DD?1*>@C9:EC?2++A3+F4EEB<E>EEIEDC2?C:;AB+==1",
+ assertEquals( "######AA>55;5(;63;;3@;A9??;6..73CDCIDA>DCB>@B=;@B?;;ADAB<DD?1*>@C9:EC?2++A3+F4EEB<E>EEIEDC2?C:;AB+==1",
getAlignment ( PrimaryOnly + ".PA.4" ).getFragmentQualities () );
}
-
+
@Test
public void Alignment_getFragmentBases_Clipped() throws ngs.ErrorMsg
{
- assertEquals( "CTTCAACATTAAAATGGAAAGCCCCATCATCCATGTGGTATATGGGCAGCCAGCAAACCTTACAAACACCACTTTTCAGATTTTCCAGAGCATCCA",
+ assertEquals( "CTTCAACATTAAAATGGAAAGCCCCATCATCCATGTGGTATATGGGCAGCCAGCAAACCTTACAAACACCACTTTTCAGATTTTCCAGAGCATCCA",
getAlignment ( PrimaryOnly + ".PA.4" ).getClippedFragmentBases () );
}
-
+
@Test
public void Alignment_getFragmentQualities_Clipped() throws ngs.ErrorMsg
{
- assertEquals( "#AA>55;5(;63;;3@;A9??;6..73CDCIDA>DCB>@B=;@B?;;ADAB<DD?1*>@C9:EC?2++A3+F4EEB<E>EEIEDC2?C:;AB+==1",
+ assertEquals( "#AA>55;5(;63;;3@;A9??;6..73CDCIDA>DCB>@B=;@B?;;ADAB<DD?1*>@C9:EC?2++A3+F4EEB<E>EEIEDC2?C:;AB+==1",
getAlignment ( PrimaryOnly + ".PA.4" ).getClippedFragmentQualities () );
}
@Test
public void Alignment_getAlignedFragmentBases() throws ngs.ErrorMsg
{
- assertEquals( "ATATGGGTTCACTCCAACAGTGAACCATTCCAAAAGACCTTGCCTGCGTGGCCATCTCCTCACAAACCCACCATCCCGCAACATCTCAGGTATCATACCTT",
+ assertEquals( "ATATGGGTTCACTCCAACAGTGAACCATTCCAAAAGACCTTGCCTGCGTGGCCATCTCCTCACAAACCCACCATCCCGCAACATCTCAGGTATCATACCTT",
getAlignment ( PrimaryOnly + ".PA.2" ).getAlignedFragmentBases () );
}
-
+
@Test
public void Alignment_getAlignmentCategory() throws ngs.ErrorMsg
{
assertEquals( ngs . Alignment . primaryAlignment, getAlignment ( PrimaryOnly + ".PA.4" ).getAlignmentCategory () );
}
-
+
@Test
public void Alignment_getAlignmentPosition() throws ngs.ErrorMsg
{
assertEquals( 85, getAlignment ( PrimaryOnly + ".PA.1" ).getAlignmentPosition () );
}
-
+
@Test
public void Alignment_getAlignmentLength() throws ngs.ErrorMsg
{
assertEquals( 101, getAlignment ( PrimaryOnly + ".PA.1" ).getAlignmentLength () );
}
-
+
@Test
public void Alignment_getIsReversedOrientation_False() throws ngs.ErrorMsg
{
@@ -405,7 +405,7 @@ public class ngs_test_CSRA1 {
{
assertTrue( getAlignment ( PrimaryOnly + ".PA.2" ).getIsReversedOrientation () );
}
-
+
@Test
public void Alignment_getSoftClip_None() throws ngs.ErrorMsg
{
@@ -427,13 +427,13 @@ public class ngs_test_CSRA1 {
assertEquals( 0, al . getSoftClip ( ngs . Alignment . clipLeft ) );
assertEquals( 13, al . getSoftClip ( ngs . Alignment . clipRight ) );
}
-
+
@Test
public void Alignment_getTemplateLength() throws ngs.ErrorMsg
{
assertEquals( 201, getAlignment ( PrimaryOnly + ".PA.1" ).getTemplateLength () );
}
-
+
@Test
public void Alignment_getShortCigar_Unclipped() throws ngs.ErrorMsg
{
@@ -455,7 +455,7 @@ public class ngs_test_CSRA1 {
{
assertEquals( "1X8=1X39=1X46=", getAlignment ( PrimaryOnly + ".PA.4" ).getLongCigar ( true ) );
}
-
+
@Test
public void Alignment_hasMate_Primary_No() throws ngs.ErrorMsg
{
@@ -485,7 +485,7 @@ public class ngs_test_CSRA1 {
getAlignment ( PrimaryOnly + ".PA.99" ) . getMateAlignmentId ( );
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
@Test
public void Alignment_getMateAlignmentId_SecondaryThrows() throws ngs.ErrorMsg
@@ -495,9 +495,9 @@ public class ngs_test_CSRA1 {
getSecondaryAlignment ( WithSecondary + ".SA.172" ) . getMateAlignmentId ( );
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
-
+
@Test
public void Alignment_getMateAlignment() throws ngs.ErrorMsg
{
@@ -506,30 +506,30 @@ public class ngs_test_CSRA1 {
@Test
public void Alignment_getMateAlignment_Missing() throws ngs.ErrorMsg
{
- try
+ try
{
getAlignment ( PrimaryOnly + ".PA.99" ) . getMateAlignment ();
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
@Test
public void Alignment_getMateAlignment_SecondaryThrows() throws ngs.ErrorMsg
{
- try
+ try
{
getSecondaryAlignment ( WithSecondary +".SA.172" ) . getMateAlignment ();
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
-
+
@Test
public void Alignment_getMateReferenceSpec() throws ngs.ErrorMsg
{
assertEquals ( "supercont2.1", getAlignment ( PrimaryOnly + ".PA.1" ) . getMateReferenceSpec () );
}
-
+
@Test
public void Alignment_getMateIsReversedOrientation_Yes() throws ngs.ErrorMsg
{
@@ -540,22 +540,22 @@ public class ngs_test_CSRA1 {
{
assertFalse( getAlignment ( PrimaryOnly + ".PA.2" ) . getMateIsReversedOrientation () );
}
-
+
@Test
public void Alignment_isPaired_MultiFragmentsPerSpot() throws ngs.ErrorMsg
{
ngs . ReadCollection readCollection = NGS . openReadCollection ( PrimaryOnly );
ngs . Alignment alignment = readCollection . getAlignment ( PrimaryOnly + ".PA.1" );
assertTrue( alignment.isPaired() );
-
+
alignment = readCollection . getAlignment ( PrimaryOnly + ".PA.2" );
assertTrue( alignment.isPaired() );
-
+
// has unaligned mate
alignment = readCollection . getAlignment ( PrimaryOnly + ".PA.6" );
assertTrue( alignment.isPaired() );
}
-
+
@Test
public void Alignment_isPaired_SingleFragmentPerSpot() throws ngs.ErrorMsg
{
@@ -576,19 +576,19 @@ public class ngs_test_CSRA1 {
{
assertEquals ( "gi|218511148|ref|NC_011752.1|", getReferenceSequence () . getCanonicalName () );
}
-
+
@Test
public void ReferenceSequence_getIsCircular_Yes() throws ngs.ErrorMsg
{
assertTrue( getReferenceSequence () . getIsCircular () );
}
-
+
@Test
public void ReferenceSequence_getLength() throws ngs.ErrorMsg
{
assertEquals ( 72482, getReferenceSequence () . getLength() );
}
-
+
@Test
public void ReferenceSequence_getReferenceBases() throws ngs.ErrorMsg
{
@@ -599,7 +599,7 @@ public class ngs_test_CSRA1 {
{
assertEquals ( "TACA", getReferenceSequence () . getReferenceBases ( 4998, 4 ) );
}
-
+
@Test
public void ReferenceSequence_getReferenceChunk() throws ngs.ErrorMsg
{
@@ -623,13 +623,13 @@ public class ngs_test_CSRA1 {
{
assertEquals ( "supercont2.1", getReference () . getCommonName () );
}
-
+
@Test
public void Reference_getCanonicalName () throws ngs.ErrorMsg
{
assertEquals ( "NC_000007.13", NGS . openReadCollection ( "SRR821492" ) . getReference ( "chr7" ) . getCanonicalName () );
}
-
+
@Test
public void Reference_getIsCircular_No() throws ngs.ErrorMsg
{
@@ -640,13 +640,13 @@ public class ngs_test_CSRA1 {
{
assertTrue( NGS . openReadCollection ( "SRR821492" ) . getReference ( "chrM" ) . getIsCircular () );
}
-
+
@Test
public void Reference_getLength() throws ngs.ErrorMsg
{
assertEquals ( 2291499l, getReference () . getLength() );
}
-
+
@Test
public void Reference_getReferenceBases() throws ngs.ErrorMsg
{
@@ -657,7 +657,7 @@ public class ngs_test_CSRA1 {
{
assertEquals ( "GCGCTATGAC", getReference () . getReferenceBases ( 9000, 10 ) );
}
-
+
@Test
public void Reference_getReferenceChunk() throws ngs.ErrorMsg
{
@@ -668,13 +668,13 @@ public class ngs_test_CSRA1 {
{
assertEquals ( "GCGCTATGAC", getReference () . getReferenceChunk ( 9000, 10 ) );
}
-
+
@Test
public void Reference_getAlignment() throws ngs.ErrorMsg
{
assertEquals ( PrimaryOnly + ".PA.1", getReference () . getAlignment ( PrimaryOnly + ".PA.1" ) . getAlignmentId () );
}
-
+
//TODO: getPileups
//TODO: getPileupSlice
//TODO: getPileupSlice_Filtered
@@ -685,14 +685,14 @@ public class ngs_test_CSRA1 {
public void ReferenceIterator_ThrowBeforeNext() throws ngs.ErrorMsg
{
ngs.ReferenceIterator it = NGS . openReadCollection ( PrimaryOnly ) . getReferences ();
- try
+ try
{
it . getCommonName ();
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
-
+
@Test
public void ReferenceIterator_Next() throws ngs.ErrorMsg
{
@@ -700,16 +700,16 @@ public class ngs_test_CSRA1 {
assertTrue ( it . nextReference () );
assertEquals ( "supercont2.1", it . getCommonName () );
}
-
+
// AlignmentIterator from Reference (ReferenceWindow)
@Test
public void ReferenceWindow () throws ngs.ErrorMsg
{
- ngs.AlignmentIterator it = NGS . openReadCollection ( WithSecondary )
+ ngs.AlignmentIterator it = NGS . openReadCollection ( WithSecondary )
. getReference ( "gi|169794206|ref|NC_010410.1|" )
- . getAlignments ( ngs . Alignment . all );
- assertTrue ( it . nextAlignment () );
-
+ . getAlignments ( ngs . Alignment . all );
+ assertTrue ( it . nextAlignment () );
+
// the first 2 secondary alignments' locations on the list: #34, #61
long count = 1;
while ( it . nextAlignment() )
@@ -718,78 +718,78 @@ public class ngs_test_CSRA1 {
break;
++count;
}
- assertEquals ( 34, count);
+ assertEquals ( 34, count);
while ( it . nextAlignment() )
{
if ( it . getAlignmentCategory() == ngs . Alignment . secondaryAlignment )
break;
++count;
}
- assertEquals ( 61, count);
+ assertEquals ( 61, count);
}
-
+
@Test
public void ReferenceWindow_Slice () throws ngs.ErrorMsg
{
- ngs.AlignmentIterator it = NGS . openReadCollection ( WithSecondary )
+ ngs.AlignmentIterator it = NGS . openReadCollection ( WithSecondary )
. getReference ( "gi|169794206|ref|NC_010410.1|" )
- . getAlignmentSlice ( 516000, 100000 );
- assertTrue ( it . nextAlignment () );
+ . getAlignmentSlice ( 516000, 100000 );
+ assertTrue ( it . nextAlignment () );
assertEquals ( WithSecondary + ".PA.33", it. getAlignmentId () );
- assertTrue ( it . nextAlignment () );
+ assertTrue ( it . nextAlignment () );
assertEquals ( WithSecondary + ".PA.34", it. getAlignmentId () );
- assertTrue ( it . nextAlignment () );
+ assertTrue ( it . nextAlignment () );
assertEquals ( WithSecondary + ".SA.169", it. getAlignmentId () ); //secondary
- assertTrue ( it . nextAlignment () );
+ assertTrue ( it . nextAlignment () );
assertEquals ( WithSecondary + ".PA.35", it. getAlignmentId () );
- assertFalse ( it . nextAlignment () );
+ assertFalse ( it . nextAlignment () );
}
-
+
@Test
public void ReferenceWindow_Slice_Filtered_Category () throws ngs.ErrorMsg
{
- ngs.AlignmentIterator it = NGS . openReadCollection ( WithSecondary )
+ ngs.AlignmentIterator it = NGS . openReadCollection ( WithSecondary )
. getReference ( "gi|169794206|ref|NC_010410.1|" )
- . getAlignmentSlice ( 516000, 100000, ngs . Alignment . primaryAlignment );
- assertTrue ( it . nextAlignment () );
+ . getAlignmentSlice ( 516000, 100000, ngs . Alignment . primaryAlignment );
+ assertTrue ( it . nextAlignment () );
assertEquals ( WithSecondary + ".PA.33", it. getAlignmentId () );
- assertTrue ( it . nextAlignment () );
+ assertTrue ( it . nextAlignment () );
assertEquals ( WithSecondary + ".PA.34", it. getAlignmentId () );
- assertTrue ( it . nextAlignment () );
+ assertTrue ( it . nextAlignment () );
assertEquals ( WithSecondary + ".PA.35", it. getAlignmentId () ); // no secondary
- assertFalse ( it . nextAlignment () );
+ assertFalse ( it . nextAlignment () );
}
-
+
@Test
public void ReferenceWindow_Slice_Filtered_Start_Within_Slice () throws ngs.ErrorMsg
{
ngs.Reference ref = NGS . openReadCollection ( WithCircularRef )
. getReference ( "NC_012920.1" );
- ngs.AlignmentIterator it = ref . getFilteredAlignmentSlice ( 0, ref.getLength(), Alignment . all, Alignment . startWithinSlice, 0 );
-
+ ngs.AlignmentIterator it = ref . getFilteredAlignmentSlice ( 0, ref.getLength(), Alignment . all, Alignment . noWraparound, 0 );
+
assertTrue ( it . nextAlignment () );
long numberOfFilteredAlignments = 1;
long lastAlignmentPosition = it.getAlignmentPosition();
while ( it . nextAlignment () ) {
long currentPosition = it.getAlignmentPosition();
-
+
String errorMsg = "Sorting violated. Last position (" + lastAlignmentPosition + ") is higher than current one (" + currentPosition + ")";
assertTrue ( errorMsg, lastAlignmentPosition <= currentPosition );
-
+
lastAlignmentPosition = currentPosition;
numberOfFilteredAlignments++;
}
-
+ assertEquals ( "numberOfFilteredAlignments ", numberOfFilteredAlignments, 12315 );
+
it = ref . getFilteredAlignmentSlice ( 0, ref.getLength(), Alignment . all, 0, 0 );
long numberOfUnfilteredAlignments = 0;
while ( it . nextAlignment () ) {
numberOfUnfilteredAlignments++;
}
-
- assertEquals ( numberOfUnfilteredAlignments, 12317 );
- assertEquals ( numberOfFilteredAlignments, 12316 );
+
+ assertEquals ( "numberOfUnfilteredAlignments ", numberOfUnfilteredAlignments, 12316 );
}
-
+
// ReadGroup
@Test
public void ReadGroup_getName () throws ngs.ErrorMsg
@@ -809,12 +809,12 @@ public class ngs_test_CSRA1 {
ngs.ReadGroup gr = NGS . openReadCollection ( WithGroups ) . getReadGroup ( "GS57510-FS3-L03" );
ngs . Statistics stats = gr . getStatistics ();
-
+
assertEquals ( 34164461870L, stats . getAsU64 ( "BASE_COUNT" ) );
assertEquals ( 34164461870L, stats . getAsU64 ( "BIO_BASE_COUNT" ) );
assertEquals ( 488063741L, stats . getAsU64 ( "SPOT_COUNT" ) );
assertEquals ( 5368875807L, stats . getAsU64 ( "SPOT_MAX" ) );
- assertEquals ( 4880812067L, stats . getAsU64 ( "SPOT_MIN" ) );
+ assertEquals ( 4880812067L, stats . getAsU64 ( "SPOT_MIN" ) );
}
/* ReadGroup no longer supports Reads
@@ -825,7 +825,7 @@ public class ngs_test_CSRA1 {
ngs.Read r = gr . getRead ( PrimaryOnly + ".R.1" );
assertEquals ( "C1ELY.6", r . getReadGroup () );
}
-
+
@Test
public void ReadGroup_getReads () throws ngs.ErrorMsg
{
@@ -833,20 +833,20 @@ public class ngs_test_CSRA1 {
ngs.ReadIterator r = gr . getReads ( ngs . Read . partiallyAligned );
}
*/
-
+
// ReadGroupIterator
@Test
public void ReadGroupIterator_ThrowBeforeNext() throws ngs.ErrorMsg
{
ngs.ReadGroupIterator it = NGS . openReadCollection ( PrimaryOnly ) . getReadGroups ();
- try
+ try
{
it . getName ();
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
-
+
@Test
public void ReadGroupIterator_Next() throws ngs.ErrorMsg
{
@@ -858,99 +858,99 @@ public class ngs_test_CSRA1 {
assertTrue ( r . nextRead () );
assertEquals ( name, r . getReadGroup () );
*/
- }
-
+ }
+
@Test
public void ReadCollection_openReadCollection_null() throws ngs.ErrorMsg
{
- try
+ try
{
NGS.openReadCollection(null);
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
-
+
@Test
public void ReadCollection_setAppVersion_null() throws ngs.ErrorMsg
{
NGS.setAppVersionString(null);
}
-
+
@Test
public void ReadCollection_openReferenceSequence_null() throws ngs.ErrorMsg
{
- try
+ try
{
NGS.openReferenceSequence(null);
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
-
+
@Test
public void ReadCollection_isValid_null() throws ngs.ErrorMsg
{
- try
+ try
{
NGS.isValid(null);
fail();
}
- catch ( NullPointerException e ) {}
+ catch ( NullPointerException e ) {}
}
-
+
@Test
public void ReadCollection_hasReadGroup_null() throws ngs.ErrorMsg
{
- try
+ try
{
ReadCollection run = NGS.openReadCollection(PrimaryOnly);
run.hasReadGroup(null);
}
- catch ( ngs.ErrorMsg e )
+ catch ( ngs.ErrorMsg e )
{
fail();
- }
+ }
}
-
+
@Test
public void ReadCollection_hasReference_null() throws ngs.ErrorMsg
{
- try
+ try
{
ReadCollection run = NGS.openReadCollection(PrimaryOnly);
run.hasReference(null);
}
- catch ( ngs.ErrorMsg e )
+ catch ( ngs.ErrorMsg e )
{
fail();
- }
+ }
}
-
+
@Test
public void ReadCollection_getReference_null() throws ngs.ErrorMsg
{
- try
+ try
{
ReadCollection run = NGS.openReadCollection(PrimaryOnly);
run.getReference(null);
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
-
+
@Test
public void ReadCollection_getAlignment_null() throws ngs.ErrorMsg
{
- try
+ try
{
ReadCollection run = NGS.openReadCollection(PrimaryOnly);
run.getAlignment(null);
fail();
}
- catch ( ngs.ErrorMsg e ) {}
+ catch ( ngs.ErrorMsg e ) {}
}
-
+
@Test
public void ReadCollection_ReadGroup_Statistics_getValueType_null() throws ngs.ErrorMsg
{
diff --git a/test/ngs/Makefile b/test/ngs/Makefile
index a479010..a9ffacd 100644
--- a/test/ngs/Makefile
+++ b/test/ngs/Makefile
@@ -37,7 +37,9 @@ TEST_TOOLS = \
test-ngs_csra1_refwin \
test-ngs_csra1_pileup \
test-ngs_reference \
+ test-ngs_byteblob \
test-ngs_fragmentblob \
+ test-ngs_referenceblob \
include $(TOP)/build/Makefile.env
@@ -158,6 +160,9 @@ TEST_NGS_CSRA1_READCOLLECTION_OBJ = \
$(TEST_BINDIR)/test-ngs_csra1_readcollection: $(TEST_NGS_CSRA1_READCOLLECTION_OBJ)
$(LP) --exe -o $@ $^ $(TEST_NGS_LIB)
+csra1rc: test-ngs_csra1_readcollection
+ $(TEST_BINDIR)/$^
+
#-------------------------------------------------------------------------------
# test-ngs_csra1_refwin
#
@@ -203,14 +208,44 @@ reference: test-ngs_reference
#-------------------------------------------------------------------------------
# test-ngs_fragmentblob
#
-TEST_NGS_SRC = \
+TEST_NGS_FRAGBLOB_SRC = \
ngstest_fragmentblob
-TEST_NGS_OBJ = \
- $(addsuffix .$(OBJX),$(TEST_NGS_SRC))
+TEST_NGS_FRAGBLOB_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_NGS_FRAGBLOB_SRC))
-$(TEST_BINDIR)/test-ngs_fragmentblob: $(TEST_NGS_OBJ) makedb
+$(TEST_BINDIR)/test-ngs_fragmentblob: $(TEST_NGS_FRAGBLOB_OBJ) makedb
$(LP) --exe -o $@ $^ $(TEST_NGS_LIB)
fragmentblob: test-ngs_fragmentblob
cd $(SRCDIR); $(TEST_BINDIR)/$^
+
+#-------------------------------------------------------------------------------
+# test-ngs_referenceblob
+#
+TEST_NGS_REFBLOB_SRC = \
+ ngstest_referenceblob
+
+TEST_NGS_REFBLOB_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_NGS_REFBLOB_SRC))
+
+$(TEST_BINDIR)/test-ngs_referenceblob: $(TEST_NGS_REFBLOB_OBJ) makedb
+ $(LP) --exe -o $@ $^ $(TEST_NGS_LIB)
+
+refblob: test-ngs_referenceblob
+ cd $(SRCDIR); $(TEST_BINDIR)/$^
+
+#-------------------------------------------------------------------------------
+# test-ngs_byteblob
+#
+TEST_NGS_BYTEBLOB_SRC = \
+ ngstest_byteblob
+
+TEST_NGS_BYTEBLOB_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_NGS_BYTEBLOB_SRC))
+
+$(TEST_BINDIR)/test-ngs_byteblob: $(TEST_NGS_BYTEBLOB_OBJ) makedb
+ $(LP) --exe -o $@ $^ $(TEST_NGS_LIB)
+
+byteblob: test-ngs_byteblob
+ cd $(SRCDIR); $(TEST_BINDIR)/$^
\ No newline at end of file
diff --git a/test/ngs/ngs_c_fixture.hpp b/test/ngs/ngs_c_fixture.hpp
index b31f183..67a3917 100644
--- a/test/ngs/ngs_c_fixture.hpp
+++ b/test/ngs/ngs_c_fixture.hpp
@@ -64,7 +64,7 @@
#define ENTRY_ACC(acc) \
ENTRY; \
m_acc = acc; \
- m_coll = NGS_ReadCollectionMake ( ctx, acc );
+ ON_FAIL ( m_coll = NGS_ReadCollectionMake ( ctx, acc ) ) { throw std :: logic_error ( string ( "NGS_ReadCollectionMake(" ) + m_acc + ") failed" ); } \
#define ENTRY_GET_READ(acc, readNo) \
ENTRY_ACC(acc); \
diff --git a/test/ngs/ngstest_byteblob.cpp b/test/ngs/ngstest_byteblob.cpp
new file mode 100644
index 0000000..4a50af8
--- /dev/null
+++ b/test/ngs/ngstest_byteblob.cpp
@@ -0,0 +1,314 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Unit tests for low-level NGS functions handling blob-bases access to REFERENCE table
+*/
+
+#include "ngs_c_fixture.hpp"
+
+#include <vdb/database.h>
+#include <vdb/blob.h>
+
+#include <NGS_Cursor.h>
+
+#include <../libs/ngs/CSRA1_Reference.h>
+#include <../libs/ngs/VByteBlob.h>
+
+#include <stdexcept>
+
+using namespace std;
+using namespace ncbi::NK;
+
+TEST_SUITE(NgsByteBlobTestSuite);
+
+const char* CSRA1_Accession = "SRR1063272";
+const char* CSRA1_Accession_WithRepeats = "SRR600094";
+
+//////////////////////////////////////////// VByteBlob
+
+class ByteBlobFixture : public NGS_C_Fixture
+{
+public:
+ ByteBlobFixture ()
+ : m_curs ( 0 ),
+ m_blob ( 0 )
+ {
+ }
+
+ virtual void Release()
+ {
+ if (m_ctx != 0)
+ {
+ if ( m_blob != 0 )
+ {
+ VBlobRelease ( m_blob );
+ }
+ if ( m_curs != 0 )
+ {
+ NGS_CursorRelease ( m_curs, m_ctx );
+ }
+ }
+ NGS_C_Fixture :: Release ();
+ }
+
+ void GetBlob ( const char* p_acc, int64_t p_firstRowId )
+ {
+ const VDatabase* db;
+ THROW_ON_RC ( VDBManagerOpenDBRead ( m_ctx -> rsrc -> vdb, & db, NULL, p_acc ) );
+ NGS_String* run_name = NGS_StringMake ( m_ctx, "", 0);
+
+ if ( m_curs != 0 )
+ {
+ NGS_CursorRelease ( m_curs, m_ctx );
+ }
+ m_curs = NGS_CursorMakeDb ( m_ctx, db, run_name, "REFERENCE", reference_col_specs, reference_NUM_COLS );
+
+ NGS_StringRelease ( run_name, m_ctx );
+ THROW_ON_RC ( VDatabaseRelease ( db ) );
+
+ if ( m_blob != 0 )
+ {
+ VBlobRelease ( m_blob );
+ }
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, p_firstRowId, reference_READ );
+ }
+
+ void CheckRange( ctx_t ctx, int64_t p_first, uint64_t p_count )
+ {
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+ int64_t first;
+ uint64_t count;
+ if ( VBlobIdRange ( m_blob, & first, & count ) != 0 )
+ {
+ throw std :: logic_error ( "VBlobRowRange() failed" );
+ }
+ if ( p_first != first || p_count != count )
+ {
+ ostringstream str;
+ str << "CheckRange(first=" << p_first << ", count=" << p_count << ") : first=" << first << ", count=" << count;
+ throw std :: logic_error ( str.str() );
+ }
+ }
+
+ const NGS_Cursor* m_curs;
+ const struct VBlob* m_blob;
+
+ const void* m_data;
+ uint64_t m_size;
+};
+
+// void VByteBlob_ContiguousChunk ( const struct VBlob* blob, ctx_t ctx, int64_t rowId, uint64_t maxRows, const void** data, uint64_t* size, bool stopAtRepeat );
+FIXTURE_TEST_CASE ( VByteBlob_BadRowId, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 0, 0, &m_data, &m_size, false );
+ REQUIRE_FAILED();
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( VByteBlob_GoodRowId, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 1, 0, &m_data, &m_size, false );
+ REQUIRE ( ! FAILED() );
+ CheckRange ( ctx, 1, 4 );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( VByteBlob_LargeRowId, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 4000, 0, &m_data, &m_size, false ); // accession row-count = 3,781
+ REQUIRE_FAILED();
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( VByteBlob_StopAtRepeat_NoRepeat, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 1, 0, &m_data, &m_size, true );
+ REQUIRE ( ! FAILED() );
+ CheckRange ( ctx, 1, 4 );
+ REQUIRE_EQ ( (uint64_t)20000, m_size );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( VByteBlob_StopAtRepeat_YesRepeat, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession_WithRepeats, 1 ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 5, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 9, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 13, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 17, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 33, reference_READ ); // this blob has 9 repeated rows, 37-45; 16 rows total
+ CheckRange ( ctx, 33, 16 ); // full blob
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 33, 0, &m_data, &m_size, true );
+ REQUIRE ( ! FAILED() );
+ REQUIRE_EQ ( (uint64_t)5*5000, m_size ); // only 5 first rows included into the contiguous chunk (up to the first repetition)
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( VByteBlob_NoStopAtRepeat_YesRepeat, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession_WithRepeats, 1 ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 5, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 9, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 13, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 17, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 33, reference_READ ); // this blob has 9 repeated rows, 37-45; 16 rows total, 8 packed
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 33, 0, &m_data, &m_size, false ); // ignore repeats
+ REQUIRE ( ! FAILED() );
+ REQUIRE_EQ ( (uint64_t)8*5000, m_size ); // all rows included: 8 = 7 unpacked + 9 packed into 1
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( VByteBlob_MaxRows_MoreThanPresent, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 1, 10, &m_data, &m_size, false );
+ REQUIRE ( ! FAILED() );
+ CheckRange ( ctx, 1, 4 );
+ REQUIRE_EQ ( (uint64_t)4*5000, m_size );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( VByteBlob_MaxRows_LessThanPresent, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 1, 2, &m_data, &m_size, false );
+ REQUIRE ( ! FAILED() );
+ REQUIRE_EQ ( (uint64_t)2*5000, m_size );
+
+ EXIT;
+}
+
+
+FIXTURE_TEST_CASE ( VByteBlob_MaxRows_LessThanPresent_WithRepeatsAfter, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession_WithRepeats, 1 ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 5, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 9, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 13, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 17, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 33, reference_READ ); // this blob has 9 repeated rows, 37-45; 16 rows total, 8 packed
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 33, 4, &m_data, &m_size, true );
+ REQUIRE ( ! FAILED() );
+ REQUIRE_EQ ( (uint64_t)4*5000, m_size ); // 4 rows (repeats come after)
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( VByteBlob_MaxRows_LessThanPresent_WithRepeatsBefore, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession_WithRepeats, 1 ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 5, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 9, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 13, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 17, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 33, reference_READ ); // this blob has 9 repeated rows, 37-45; 16 rows total, 8 packed
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 33, 14, &m_data, &m_size, true );
+ REQUIRE ( ! FAILED() );
+ REQUIRE_EQ ( (uint64_t)5*5000, m_size ); // only 5 first rows included into the contiguous chunk (up to the first repetition)
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( VByteBlob_MaxRows_LessThanPresent_WithRepeatsOverlapping, ByteBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession_WithRepeats, 1 ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 5, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 9, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 13, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 17, reference_READ ); VBlobRelease ( m_blob );
+ m_blob = NGS_CursorGetVBlob ( m_curs, m_ctx, 33, reference_READ ); // this blob has 9 repeated rows, 37-45; 16 rows total, 8 packed
+
+ VByteBlob_ContiguousChunk ( m_blob, ctx, 33, 7, &m_data, &m_size, true ); // 4 rows and 3 of the repeated
+ REQUIRE ( ! FAILED() );
+ REQUIRE_EQ ( (uint64_t)5*5000, m_size ); // only 5 first rows included into the contiguous chunk (up to the first repetition)
+
+ EXIT;
+}
+
+//////////////////////////////////////////// Main
+
+extern "C"
+{
+
+#include <kapp/args.h>
+
+ver_t CC KAppVersion ( void )
+{
+ return 0x1000000;
+}
+rc_t CC UsageSummary (const char * progname)
+{
+ return 0;
+}
+
+rc_t CC Usage ( const Args * args )
+{
+ return 0;
+}
+
+const char UsageDefaultName[] = "test-ngs_byteblob";
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ rc_t m_coll=NgsByteBlobTestSuite(argc, argv);
+ return m_coll;
+}
+
+}
+
diff --git a/test/ngs/ngstest_csra1.cpp b/test/ngs/ngstest_csra1.cpp
index 643c651..e4bf383 100644
--- a/test/ngs/ngstest_csra1.cpp
+++ b/test/ngs/ngstest_csra1.cpp
@@ -39,8 +39,6 @@
#include <vdb/manager.h>
#include <vdb/vdb-priv.h>
-#include "CSRA1_Reference.h"
-#include "NGS_Pileup.h"
#include "NGS_FragmentBlobIterator.h"
#include "NGS_FragmentBlob.h"
@@ -693,506 +691,6 @@ FIXTURE_TEST_CASE(CSRA1_NGS_AlignmentGetMateIsReversedOrientation_False, CSRA1_F
EXIT;
}
-// NGS_Reference
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetCommonName, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1");
- REQUIRE_STRING ( "supercont2.1", NGS_ReferenceGetCommonName ( m_ref, ctx ) );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetCanonicalName, CSRA1_Fixture)
-{
- ENTRY_GET_REF( "SRR821492", "chr7" );
- const char* canoName = "NC_000007.13";
- REQUIRE_STRING ( canoName, NGS_ReferenceGetCanonicalName ( m_ref, ctx ) );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIsCircular_Yes, CSRA1_Fixture)
-{
- ENTRY_GET_REF( "SRR821492", "chrM" );
- REQUIRE ( NGS_ReferenceGetIsCircular ( m_ref, ctx ) );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIsCircular_No, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- REQUIRE ( ! NGS_ReferenceGetIsCircular ( m_ref, ctx ) );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetLength, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- REQUIRE_EQ ( (uint64_t)2291499l, NGS_ReferenceGetLength ( m_ref, ctx ) );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFirstRowId, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- REQUIRE_EQ ( (int64_t)1, CSRA1_Reference_GetFirstRowId ( m_ref, ctx ) );
- EXIT;
-}
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetLastRowId, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- REQUIRE_EQ ( (int64_t)459, CSRA1_Reference_GetLastRowId ( m_ref, ctx ) );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBases_Full, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- string bases = toString ( NGS_ReferenceGetBases( m_ref, ctx, 0, -1 ), ctx, true );
- REQUIRE_EQ ( NGS_ReferenceGetLength ( m_ref, ctx ), ( uint64_t ) bases. size () );
- EXIT;
-}
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBases, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- REQUIRE_STRING ( "CCTGTCC", NGS_ReferenceGetBases( m_ref, ctx, 7000, 7 ) );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignment_Primary, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
-
- m_align = NGS_ReferenceGetAlignment ( m_ref, ctx, ( string ( CSRA1_PrimaryOnly ) + ".PA.1" ) . c_str () );
- REQUIRE ( ! FAILED () && m_align );
-
- REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".PA.1", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_PrimaryOnly_Primary, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
- REQUIRE ( ! FAILED () && m_align );
-
- REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
- REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".PA.1", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_PrimaryOnly_Secondary, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, false, true );
- REQUIRE ( ! FAILED () && m_align );
- REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_WithSecondary_Primary, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false);
- REQUIRE ( ! FAILED () && m_align );
- REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
- REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( string ( CSRA1_WithSecondary ) + ".PA.1", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
-
- EXIT;
-}
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_WithSecondary_Secondary, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, false, true );
- REQUIRE ( ! FAILED () && m_align );
- REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
- REQUIRE_STRING ( string ( CSRA1_WithSecondary ) + ".SA.169", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
-
- EXIT;
-}
-
-
-// ReferenceGetAlignments on circular references
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_Circular_Wraparound, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_WithCircularReference, "NC_012920.1" );
-
- const bool wants_primary = true;
- const bool wants_secondary = true;
- const uint32_t no_filters = 0;
- const int32_t no_map_qual = 0;
-
- m_align = NGS_ReferenceGetFilteredAlignments ( m_ref, ctx, wants_primary, wants_secondary, no_filters, no_map_qual );
- REQUIRE ( ! FAILED () && m_align );
- REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
- // by default, the first returned alignment starts before the start of the circular reference
- REQUIRE_EQ ( (int64_t)16477, NGS_AlignmentGetAlignmentPosition ( m_align, ctx ) );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_Circular_NoWraparound, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_WithCircularReference, "NC_012920.1" );
-
- const bool wants_primary = true;
- const bool wants_secondary = true;
- const uint32_t filters = NGS_AlignmentFilterBits_no_wraparound;
- const int32_t no_map_qual = 0;
-
- m_align = NGS_ReferenceGetFilteredAlignments ( m_ref, ctx, wants_primary, wants_secondary, filters, no_map_qual );
- REQUIRE ( ! FAILED () && m_align );
- REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
- // with a no-wraparound filter, the first returned alignment starts at/after the start of the circular reference
- REQUIRE_EQ ( (int64_t)5, NGS_AlignmentGetAlignmentPosition ( m_align, ctx ) );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_FullReference_Wraparound_Count, CSRA1_Fixture )
-{
- ENTRY_GET_REF( CSRA1_WithCircularReference, "NC_012920.1" );
-
- const bool wants_primary = true;
- const bool wants_secondary = true;
- const uint32_t no_filters = 0;
- const int32_t no_map_qual = 0;
-
- m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, 0, NGS_ReferenceGetLength ( m_ref, ctx ), wants_primary, wants_secondary, no_filters, no_map_qual );
- REQUIRE ( ! FAILED () && m_align );
-
- uint64_t count = 0;
- while ( NGS_AlignmentIteratorNext ( m_align, ctx ) )
- {
- ++count;
- }
- REQUIRE_EQ ( (uint64_t) 12317, count );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_FullReference_NoWraparound_Count, CSRA1_Fixture )
-{
- ENTRY_GET_REF( CSRA1_WithCircularReference, "NC_012920.1" );
-
- const bool wants_primary = true;
- const bool wants_secondary = true;
- const uint32_t filters = NGS_AlignmentFilterBits_no_wraparound;
- const int32_t no_map_qual = 0;
-
- m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, 0, NGS_ReferenceGetLength ( m_ref, ctx ), wants_primary, wants_secondary, filters, no_map_qual );
- REQUIRE ( ! FAILED () && m_align );
-
- int64_t lastPos = 0;
- uint64_t count = 0;
- while ( NGS_AlignmentIteratorNext ( m_align, ctx ) )
- {
- ++count;
- int64_t newPos = NGS_AlignmentGetAlignmentPosition ( m_align, ctx );
- REQUIRE_LE ( lastPos, newPos );
- lastPos = newPos;
- }
- REQUIRE_EQ ( (uint64_t) 12316, count );
-
- EXIT;
-}
-
-// NGS_ReferenceGetChunk
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetChunk_Empty, CSRA1_Fixture)
-{ // offset beyond the end of reference
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- REQUIRE_STRING ( "", NGS_ReferenceGetChunk ( m_ref, ctx, 30000000, 10) );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_All, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 0, (size_t)-1 );
- REQUIRE_EQ( (size_t)20000, NGS_StringSize ( chunk, ctx ) );
-
- string str = string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
- REQUIRE_EQ( string("GAATTCT"), str . substr (0, 7) );
- REQUIRE_EQ( string("CATCA"), str . substr ( str.size() - 5, 5 ) );
-
- NGS_StringRelease ( chunk, ctx );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Offset_1, CSRA1_Fixture)
-{ // offset points into the first blob of REFERENCE.READ
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 1000, (size_t)-1 );
- REQUIRE_EQ( (size_t)19000, NGS_StringSize ( chunk, ctx ) ); // first blob's size is 20000
-
- string str = string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
- REQUIRE_EQ( string("TCCATTC"), str . substr (0, 7) );
- REQUIRE_EQ( string("CATCA"), str . substr (str.size() - 5, 5) );
-
- NGS_StringRelease ( chunk, ctx );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Offset_2, CSRA1_Fixture)
-{ // offset points into the second blob of REFERENCE.READ
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 22000, (size_t)-1 );
- REQUIRE_EQ( (size_t)3000, NGS_StringSize ( chunk, ctx ) ); // second blob's size is 5000
-
- string str = string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
- REQUIRE_EQ( string("CTCAGAT"), str . substr (0, 7) );
- REQUIRE_EQ( string("TATTC"), str . substr (str.size() - 5, 5) );
-
- NGS_StringRelease ( chunk, ctx );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_1, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
-
- NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 2000, 10 );
- REQUIRE_EQ( string ( "GGGCAAATGA" ), string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) ) );
- NGS_StringRelease ( chunk, ctx );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_2, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
-
- NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 20020, 10 );
- REQUIRE_EQ( string ( "ACATGACGGA" ), string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) ) );
- NGS_StringRelease ( chunk, ctx );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_ReturnShorter, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
-
- NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 19995, 200 ); // only 5 bases left in this blob
- REQUIRE_EQ( string ( "CATCA" ), string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) ) );
- NGS_StringRelease ( chunk, ctx );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Chunks_Vs_Blobs, CSRA1_Fixture)
-{
- ENTRY_GET_REF( "SRR600094", "NC_000022.10" );
-
- NGS_String * full = NGS_ReferenceGetBases ( m_ref, ctx, 0, (uint64_t)-1 );
- REQUIRE ( ! FAILED () );
- REQUIRE_NOT_NULL ( full );
- REQUIRE_EQ ( (uint64_t)NGS_ReferenceGetLength ( m_ref, ctx ), (uint64_t)NGS_StringSize ( full, ctx ) );
-
- size_t offset=0;
- while ( offset < NGS_StringSize ( full, ctx ) )
- {
- NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, offset, 5000 );
- REQUIRE ( ! FAILED () );
- REQUIRE_NOT_NULL ( chunk );
- size_t chunkSize = NGS_StringSize ( chunk, ctx );
- REQUIRE_EQ ( string ( NGS_StringData ( full, ctx ) + offset, chunkSize ), string ( NGS_StringData ( chunk, ctx ), chunkSize ) );
- NGS_StringRelease ( chunk, ctx );
- offset += chunkSize;
- }
-
- NGS_StringRelease ( full, ctx );
-
- EXIT;
-}
-
-//
-
-FIXTURE_TEST_CASE(CSRA1_NGS_Reference_SharedCursor, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- NGS_Reference* ref2 = NGS_ReadCollectionGetReference ( m_coll, ctx, "supercont2.2" );
-
- string name = toString ( NGS_ReferenceGetCommonName ( m_ref, ctx), ctx, true );
- string name2 = toString ( NGS_ReferenceGetCommonName ( ref2, ctx), ctx, true );
-
- REQUIRE_NE ( name, name2 );
-
- NGS_ReferenceRelease ( ref2, m_ctx );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetStats, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
-
- NGS_Statistics * stats = NGS_ReferenceGetStatistics ( m_ref, ctx );
- REQUIRE ( ! FAILED () );
-
- // Reference stats are empty for now
- const char* path;
- REQUIRE ( ! NGS_StatisticsNextPath ( stats, ctx, "", &path ) );
-
- NGS_StatisticsRelease ( stats, ctx );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetPileups, CSRA1_Fixture)
-{
- ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
-
- NGS_Pileup* pileup = NGS_ReferenceGetPileups( m_ref, ctx, true, false);
- REQUIRE ( ! FAILED () && pileup );
-
- NGS_PileupRelease ( pileup, ctx );
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetPileupSlice, CSRA1_Fixture)
-{
- ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
-
- NGS_Pileup* pileup = NGS_ReferenceGetPileupSlice( m_ref, ctx, 500, 10, true, false);
- REQUIRE ( ! FAILED () && pileup );
-
- NGS_PileupRelease ( pileup, ctx );
- EXIT;
-}
-
-// Iteration over References
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIterator_GetLength_1, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
-
- NGS_Reference* refIt = NGS_ReadCollectionGetReferences ( m_coll, m_ctx );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE ( ! FAILED () );
-
- REQUIRE_EQ ( NGS_ReferenceGetLength ( refIt, ctx ), NGS_ReferenceGetLength ( m_ref, ctx ) );
- REQUIRE ( ! FAILED () );
-
- NGS_ReferenceRelease ( refIt, ctx );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIterator_GetLength_2, CSRA1_Fixture)
-{ // bug report: after a 1-chunk reference, the next reference in an iterator report wrong length
- ENTRY_ACC( "SRR1121656" );
- m_ref = NGS_ReadCollectionGetReferences ( m_coll, m_ctx );
-
- bool checked = false;
- while ( NGS_ReferenceIteratorNext ( m_ref, ctx ) )
- {
- if ( string ( "GL000207.1" ) == toString ( NGS_ReferenceGetCommonName ( m_ref, ctx ), ctx, true ) )
- {
- REQUIRE_EQ ( (uint64_t)4262, NGS_ReferenceGetLength ( m_ref, ctx ) );
- }
- else if ( string ( "GL000226.1" ) == toString ( NGS_ReferenceGetCommonName ( m_ref, ctx ), ctx, true ) )
- {
- REQUIRE_EQ ( (uint64_t)15008, NGS_ReferenceGetLength ( m_ref, ctx ) );
- checked = true;
- break;
- }
- }
- REQUIRE ( checked );
-
- EXIT;
-}
-
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIterator_GetFirstRowId, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- NGS_Reference* refIt = NGS_ReadCollectionGetReferences ( m_coll, m_ctx );
-
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)1, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)460, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)785, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)1101, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)1318, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)1681, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)1966, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)2246, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)2526, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)2764, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)2976, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)3289, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)3444, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)3596, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
- REQUIRE ( ! NGS_ReferenceIteratorNext ( refIt, ctx ) );
-
- NGS_ReferenceRelease ( refIt, ctx );
- EXIT;
-}
-FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIterator_GetLastRowId, CSRA1_Fixture)
-{
- ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
- NGS_Reference* refIt = NGS_ReadCollectionGetReferences ( m_coll, m_ctx );
-
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)459, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)784, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)1100, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)1317, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)1680, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)1965, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)2245, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)2525, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)2763, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)2975, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)3288, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)3443, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)3595, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
- REQUIRE_EQ ( (int64_t)3781, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
- REQUIRE ( ! NGS_ReferenceIteratorNext ( refIt, ctx ) );
-
- NGS_ReferenceRelease ( refIt, ctx );
- EXIT;
-}
-
-
-
// ReadGroups
FIXTURE_TEST_CASE(CSRA1_NGS_ReadGroupGetName, CSRA1_Fixture)
{
diff --git a/test/ngs/ngstest_fragmentblob.cpp b/test/ngs/ngstest_fragmentblob.cpp
index 54e9991..2dfe69d 100644
--- a/test/ngs/ngstest_fragmentblob.cpp
+++ b/test/ngs/ngstest_fragmentblob.cpp
@@ -172,7 +172,7 @@ FIXTURE_TEST_CASE ( NGS_FragmentBlob_RowRange, FragmentBlobFixture )
EXIT;
}
-FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_DuplicateRelease, FragmentBlobFixture )
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_DuplicateRelease, FragmentBlobFixture )
{
ENTRY;
MakeBlob ( SRA_Accession, 1 );
@@ -186,7 +186,7 @@ FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_DuplicateRelease, FragmentBlobFixture )
EXIT;
}
-FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_Data_BadArg, FragmentBlobFixture )
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_Data_BadArg, FragmentBlobFixture )
{
ENTRY;
@@ -195,7 +195,7 @@ FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_Data_BadArg, FragmentBlobFixture )
EXIT;
}
-FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_Data, FragmentBlobFixture )
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_Data, FragmentBlobFixture )
{
ENTRY;
MakeBlob ( SRA_Accession, 1 );
@@ -207,7 +207,7 @@ FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_Data, FragmentBlobFixture )
EXIT;
}
-FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_Size_BadArg, FragmentBlobFixture )
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_Size_BadArg, FragmentBlobFixture )
{
ENTRY;
@@ -216,7 +216,7 @@ FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_Size_BadArg, FragmentBlobFixture )
EXIT;
}
-FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_Size, FragmentBlobFixture )
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_Size, FragmentBlobFixture )
{
ENTRY;
MakeBlob ( SRA_Accession, 1 );
@@ -226,7 +226,18 @@ FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_Size, FragmentBlobFixture )
EXIT;
}
-FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_InfoByOffset_BadSelf, FragmentBlobFixture )
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_Run, FragmentBlobFixture )
+{
+ ENTRY;
+ MakeBlob ( SRA_Accession, 1 );
+
+ const NGS_String * run = NGS_FragmentBlobRun ( m_blob, m_ctx );
+ REQUIRE_EQ ( string ( SRA_Accession ), string ( NGS_StringData ( run, m_ctx ) , NGS_StringSize ( run, m_ctx ) ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_InfoByOffset_BadSelf, FragmentBlobFixture )
{
ENTRY;
@@ -241,7 +252,7 @@ FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_InfoByOffset_BadSelf, FragmentBlobFixtu
}
// TODO: NULL for optional parameters
-FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_InfoByOffset_Biological, FragmentBlobFixture )
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_InfoByOffset_Biological, FragmentBlobFixture )
{
ENTRY;
MakeBlob ( SRA_Accession, 1 );
@@ -265,7 +276,7 @@ FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_InfoByOffset_Biological, FragmentBlobFi
EXIT;
}
-FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_InfoByOffset_Technical, FragmentBlobFixture )
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_InfoByOffset_Technical, FragmentBlobFixture )
{
ENTRY;
MakeBlob ( SRA_Accession, 1 );
@@ -289,7 +300,7 @@ FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_InfoByOffset_Technical, FragmentBlobFix
EXIT;
}
-FIXTURE_TEST_CASE ( NGS_FragmentBlobMake_InfoByOffset_WithRepeat, FragmentBlobFixture )
+FIXTURE_TEST_CASE ( NGS_FragmentBlob_InfoByOffset_WithRepeat, FragmentBlobFixture )
{ // VDB-3026, VDB-2809
ENTRY;
diff --git a/test/ngs/ngstest_reference.cpp b/test/ngs/ngstest_reference.cpp
index 54dfa00..de6e524 100644
--- a/test/ngs/ngstest_reference.cpp
+++ b/test/ngs/ngstest_reference.cpp
@@ -36,14 +36,593 @@
#include <kfc/xc.h>
+#include "NGS_Pileup.h"
#include "NGS_ReferenceSequence.h"
#include "NGS_String.h"
+#include "CSRA1_Reference.h"
+#include "NGS_ReferenceBlobIterator.h"
+#include "NGS_ReferenceBlob.h"
using namespace std;
using namespace ncbi::NK;
TEST_SUITE(NgsReferenceTestSuite);
+const char* CSRA1_PrimaryOnly = "SRR1063272";
+const char* CSRA1_WithSecondary = "SRR833251";
+const char* CSRA1_WithCircularReference = "SRR1769246";
+
+class CSRA1_ReferenceFixture : public NGS_C_Fixture
+{
+public:
+ CSRA1_ReferenceFixture()
+ : m_align(0)
+ {
+ }
+ ~CSRA1_ReferenceFixture()
+ {
+ }
+
+ virtual void Release()
+ {
+ if (m_ctx != 0)
+ {
+ if (m_align != 0)
+ {
+ NGS_AlignmentRelease ( m_align, m_ctx );
+ }
+ }
+ NGS_C_Fixture :: Release ();
+ }
+
+ NGS_Alignment* m_align;
+};
+
+// NGS_Reference
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetCommonName, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1");
+ REQUIRE_STRING ( "supercont2.1", NGS_ReferenceGetCommonName ( m_ref, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetCanonicalName, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( "SRR821492", "chr7" );
+ const char* canoName = "NC_000007.13";
+ REQUIRE_STRING ( canoName, NGS_ReferenceGetCanonicalName ( m_ref, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIsCircular_Yes, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( "SRR821492", "chrM" );
+ REQUIRE ( NGS_ReferenceGetIsCircular ( m_ref, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIsCircular_No, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ REQUIRE ( ! NGS_ReferenceGetIsCircular ( m_ref, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetLength, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ REQUIRE_EQ ( (uint64_t)2291499l, NGS_ReferenceGetLength ( m_ref, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFirstRowId, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ REQUIRE_EQ ( (int64_t)1, CSRA1_Reference_GetFirstRowId ( m_ref, ctx ) );
+ EXIT;
+}
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetLastRowId, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ REQUIRE_EQ ( (int64_t)459, CSRA1_Reference_GetLastRowId ( m_ref, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBases_Full, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ string bases = toString ( NGS_ReferenceGetBases( m_ref, ctx, 0, -1 ), ctx, true );
+ REQUIRE_EQ ( NGS_ReferenceGetLength ( m_ref, ctx ), ( uint64_t ) bases. size () );
+ EXIT;
+}
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBases, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ REQUIRE_STRING ( "CCTGTCC", NGS_ReferenceGetBases( m_ref, ctx, 7000, 7 ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignment_Primary, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ m_align = NGS_ReferenceGetAlignment ( m_ref, ctx, ( string ( CSRA1_PrimaryOnly ) + ".PA.1" ) . c_str () );
+ REQUIRE ( ! FAILED () && m_align );
+
+ REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".PA.1", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_PrimaryOnly_Primary, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
+ REQUIRE ( ! FAILED () && m_align );
+
+ REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
+ REQUIRE ( ! FAILED () );
+
+ REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".PA.1", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_PrimaryOnly_Secondary, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, false, true );
+ REQUIRE ( ! FAILED () && m_align );
+ REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_WithSecondary_Primary, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false);
+ REQUIRE ( ! FAILED () && m_align );
+ REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
+ REQUIRE ( ! FAILED () );
+
+ REQUIRE_STRING ( string ( CSRA1_WithSecondary ) + ".PA.1", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
+
+ EXIT;
+}
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_WithSecondary_Secondary, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, false, true );
+ REQUIRE ( ! FAILED () && m_align );
+ REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
+
+ REQUIRE_STRING ( string ( CSRA1_WithSecondary ) + ".SA.169", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
+
+ EXIT;
+}
+
+
+// ReferenceGetAlignments on circular references
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_Circular_Wraparound, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_WithCircularReference, "NC_012920.1" );
+
+ const bool wants_primary = true;
+ const bool wants_secondary = true;
+ const uint32_t no_filters = 0;
+ const int32_t no_map_qual = 0;
+
+ m_align = NGS_ReferenceGetFilteredAlignments ( m_ref, ctx, wants_primary, wants_secondary, no_filters, no_map_qual );
+ REQUIRE ( ! FAILED () && m_align );
+ REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
+
+ // by default, the first returned alignment starts before the start of the circular reference
+ REQUIRE_EQ ( (int64_t)16477, NGS_AlignmentGetAlignmentPosition ( m_align, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignments_Circular_NoWraparound, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_WithCircularReference, "NC_012920.1" );
+
+ const bool wants_primary = true;
+ const bool wants_secondary = true;
+ const uint32_t filters = NGS_AlignmentFilterBits_no_wraparound;
+ const int32_t no_map_qual = 0;
+
+ m_align = NGS_ReferenceGetFilteredAlignments ( m_ref, ctx, wants_primary, wants_secondary, filters, no_map_qual );
+ REQUIRE ( ! FAILED () && m_align );
+ REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
+
+ // with a no-wraparound filter, the first returned alignment starts at/after the start of the circular reference
+ REQUIRE_EQ ( (int64_t)5, NGS_AlignmentGetAlignmentPosition ( m_align, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetAlignmentsSlice_Circular_Wraparound, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_WithCircularReference, "NC_012920.1" );
+
+ const bool wants_primary = true;
+ const bool wants_secondary = true;
+ const uint32_t no_filters = 0;
+ const int32_t no_map_qual = 0;
+
+ m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, 16565, 6, wants_primary, wants_secondary, no_filters, no_map_qual ); /* ref lingth = 16569; this slice wraps around */
+ REQUIRE ( ! FAILED () && m_align );
+
+ REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
+ REQUIRE_EQ ( (int64_t)16477, NGS_AlignmentGetAlignmentPosition ( m_align, ctx ) ); // SRR1769246.PA.3275490, an actual wraparound
+ REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
+ REQUIRE_EQ ( (int64_t)16472, NGS_AlignmentGetAlignmentPosition ( m_align, ctx ) ); // SRR1769246.PA.3275487, overlaps with the slice but does not wrap around
+ REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_FullReference_Wraparound_Count, CSRA1_ReferenceFixture )
+{
+ ENTRY_GET_REF( CSRA1_WithCircularReference, "NC_012920.1" );
+
+ const bool wants_primary = true;
+ const bool wants_secondary = true;
+ const uint32_t no_filters = 0;
+ const int32_t no_map_qual = 0;
+
+ m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, 0, NGS_ReferenceGetLength ( m_ref, ctx ), wants_primary, wants_secondary, no_filters, no_map_qual );
+ REQUIRE ( ! FAILED () && m_align );
+
+ uint64_t count = 0;
+ while ( NGS_AlignmentIteratorNext ( m_align, ctx ) )
+ {
+ ++count;
+ }
+ REQUIRE_EQ ( (uint64_t) 12316, count );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_FullReference_NoWraparound_Count, CSRA1_ReferenceFixture )
+{
+ ENTRY_GET_REF( CSRA1_WithCircularReference, "NC_012920.1" );
+
+ const bool wants_primary = true;
+ const bool wants_secondary = true;
+ const uint32_t filters = NGS_AlignmentFilterBits_no_wraparound;
+ const int32_t no_map_qual = 0;
+
+ m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, 0, NGS_ReferenceGetLength ( m_ref, ctx ), wants_primary, wants_secondary, filters, no_map_qual );
+ REQUIRE ( ! FAILED () && m_align );
+
+ int64_t lastPos = 0;
+ uint64_t count = 0;
+ while ( NGS_AlignmentIteratorNext ( m_align, ctx ) )
+ {
+ ++count;
+ int64_t newPos = NGS_AlignmentGetAlignmentPosition ( m_align, ctx );
+ REQUIRE_LE ( lastPos, newPos );
+ lastPos = newPos;
+ }
+ REQUIRE_EQ ( (uint64_t) 12315, count ); // does not include SRR1769246.PA.3275490, the only wraparound
+
+ EXIT;
+}
+
+// NGS_ReferenceGetChunk
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetChunk_Empty, CSRA1_ReferenceFixture)
+{ // offset beyond the end of reference
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ REQUIRE_STRING ( "", NGS_ReferenceGetChunk ( m_ref, ctx, 30000000, 10) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_All, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 0, (size_t)-1 );
+ REQUIRE_EQ( (size_t)20000, NGS_StringSize ( chunk, ctx ) );
+
+ string str = string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
+ REQUIRE_EQ( string("GAATTCT"), str . substr (0, 7) );
+ REQUIRE_EQ( string("CATCA"), str . substr ( str.size() - 5, 5 ) );
+
+ NGS_StringRelease ( chunk, ctx );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Offset_1, CSRA1_ReferenceFixture)
+{ // offset points into the first blob of REFERENCE.READ
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 1000, (size_t)-1 );
+ REQUIRE_EQ( (size_t)19000, NGS_StringSize ( chunk, ctx ) ); // first blob's size is 20000
+
+ string str = string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
+ REQUIRE_EQ( string("TCCATTC"), str . substr (0, 7) );
+ REQUIRE_EQ( string("CATCA"), str . substr (str.size() - 5, 5) );
+
+ NGS_StringRelease ( chunk, ctx );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Offset_2, CSRA1_ReferenceFixture)
+{ // offset points into the second blob of REFERENCE.READ
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 22000, (size_t)-1 );
+ REQUIRE_EQ( (size_t)3000, NGS_StringSize ( chunk, ctx ) ); // second blob's size is 5000
+
+ string str = string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
+ REQUIRE_EQ( string("CTCAGAT"), str . substr (0, 7) );
+ REQUIRE_EQ( string("TATTC"), str . substr (str.size() - 5, 5) );
+
+ NGS_StringRelease ( chunk, ctx );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_1, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 2000, 10 );
+ REQUIRE_EQ( string ( "GGGCAAATGA" ), string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) ) );
+ NGS_StringRelease ( chunk, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_2, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 20020, 10 );
+ REQUIRE_EQ( string ( "ACATGACGGA" ), string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) ) );
+ NGS_StringRelease ( chunk, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_ReturnShorter, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 19995, 200 ); // only 5 bases left in this blob
+ REQUIRE_EQ( string ( "CATCA" ), string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) ) );
+ NGS_StringRelease ( chunk, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Chunks_Vs_Blobs, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( "SRR600094", "NC_000022.10" );
+
+ NGS_String * full = NGS_ReferenceGetBases ( m_ref, ctx, 0, (uint64_t)-1 );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( full );
+ REQUIRE_EQ ( (uint64_t)NGS_ReferenceGetLength ( m_ref, ctx ), (uint64_t)NGS_StringSize ( full, ctx ) );
+
+ size_t offset=0;
+ while ( offset < NGS_StringSize ( full, ctx ) )
+ {
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, offset, 5000 );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( chunk );
+ size_t chunkSize = NGS_StringSize ( chunk, ctx );
+ REQUIRE_EQ ( string ( NGS_StringData ( full, ctx ) + offset, chunkSize ), string ( NGS_StringData ( chunk, ctx ), chunkSize ) );
+ NGS_StringRelease ( chunk, ctx );
+ offset += chunkSize;
+ }
+
+ NGS_StringRelease ( full, ctx );
+
+ EXIT;
+}
+
+//
+
+FIXTURE_TEST_CASE(CSRA1_NGS_Reference_SharedCursor, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ NGS_Reference* ref2 = NGS_ReadCollectionGetReference ( m_coll, ctx, "supercont2.2" );
+
+ string name = toString ( NGS_ReferenceGetCommonName ( m_ref, ctx), ctx, true );
+ string name2 = toString ( NGS_ReferenceGetCommonName ( ref2, ctx), ctx, true );
+
+ REQUIRE_NE ( name, name2 );
+
+ NGS_ReferenceRelease ( ref2, m_ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetStats, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_Statistics * stats = NGS_ReferenceGetStatistics ( m_ref, ctx );
+ REQUIRE ( ! FAILED () );
+
+ // Reference stats are empty for now
+ const char* path;
+ REQUIRE ( ! NGS_StatisticsNextPath ( stats, ctx, "", &path ) );
+
+ NGS_StatisticsRelease ( stats, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetPileups, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_Pileup* pileup = NGS_ReferenceGetPileups( m_ref, ctx, true, false);
+ REQUIRE ( ! FAILED () && pileup );
+
+ NGS_PileupRelease ( pileup, ctx );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetPileupSlice, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_Pileup* pileup = NGS_ReferenceGetPileupSlice( m_ref, ctx, 500, 10, true, false);
+ REQUIRE ( ! FAILED () && pileup );
+
+ NGS_PileupRelease ( pileup, ctx );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBlobs, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_ReferenceBlobIterator* blobIt = NGS_ReferenceGetBlobs( m_ref, ctx, 0, (uint64_t)-1 );
+ REQUIRE ( ! FAILED () && blobIt );
+
+ REQUIRE ( NGS_ReferenceBlobIteratorHasMore ( blobIt, ctx ) );
+ REQUIRE ( ! FAILED () );
+ struct NGS_ReferenceBlob* blob = NGS_ReferenceBlobIteratorNext ( blobIt, ctx );
+ REQUIRE ( ! FAILED () && blob );
+
+ int64_t first;
+ uint64_t count;
+ NGS_ReferenceBlobRowRange ( blob, ctx, & first, & count );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_EQ ( (int64_t)1, first );
+ REQUIRE_EQ ( (uint64_t)4, count );
+
+ NGS_ReferenceBlobRelease ( blob, ctx );
+ NGS_ReferenceBlobIteratorRelease ( blobIt, ctx );
+ EXIT;
+}
+
+// Iteration over References
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIterator_GetLength_1, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_Reference* refIt = NGS_ReadCollectionGetReferences ( m_coll, m_ctx );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE ( ! FAILED () );
+
+ REQUIRE_EQ ( NGS_ReferenceGetLength ( refIt, ctx ), NGS_ReferenceGetLength ( m_ref, ctx ) );
+ REQUIRE ( ! FAILED () );
+
+ NGS_ReferenceRelease ( refIt, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIterator_GetLength_2, CSRA1_ReferenceFixture)
+{ // bug report: after a 1-chunk reference, the next reference in an iterator report wrong length
+ ENTRY_ACC( "SRR1121656" );
+ m_ref = NGS_ReadCollectionGetReferences ( m_coll, m_ctx );
+
+ bool checked = false;
+ while ( NGS_ReferenceIteratorNext ( m_ref, ctx ) )
+ {
+ if ( string ( "GL000207.1" ) == toString ( NGS_ReferenceGetCommonName ( m_ref, ctx ), ctx, true ) )
+ {
+ REQUIRE_EQ ( (uint64_t)4262, NGS_ReferenceGetLength ( m_ref, ctx ) );
+ }
+ else if ( string ( "GL000226.1" ) == toString ( NGS_ReferenceGetCommonName ( m_ref, ctx ), ctx, true ) )
+ {
+ REQUIRE_EQ ( (uint64_t)15008, NGS_ReferenceGetLength ( m_ref, ctx ) );
+ checked = true;
+ break;
+ }
+ }
+ REQUIRE ( checked );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIterator_GetFirstRowId, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ NGS_Reference* refIt = NGS_ReadCollectionGetReferences ( m_coll, m_ctx );
+
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)1, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)460, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)785, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)1101, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)1318, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)1681, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)1966, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)2246, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)2526, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)2764, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)2976, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)3289, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)3444, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)3596, CSRA1_Reference_GetFirstRowId ( refIt, ctx ) );
+ REQUIRE ( ! NGS_ReferenceIteratorNext ( refIt, ctx ) );
+
+ NGS_ReferenceRelease ( refIt, ctx );
+ EXIT;
+}
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceIterator_GetLastRowId, CSRA1_ReferenceFixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ NGS_Reference* refIt = NGS_ReadCollectionGetReferences ( m_coll, m_ctx );
+
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)459, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)784, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)1100, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)1317, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)1680, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)1965, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)2245, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)2525, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)2763, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)2975, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)3288, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)3443, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)3595, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( NGS_ReferenceIteratorNext ( refIt, ctx ) );
+ REQUIRE_EQ ( (int64_t)3781, CSRA1_Reference_GetLastRowId ( refIt, ctx ) );
+ REQUIRE ( ! NGS_ReferenceIteratorNext ( refIt, ctx ) );
+
+ NGS_ReferenceRelease ( refIt, ctx );
+ EXIT;
+}
+
+// NGS_ReferenceSequence
+
FIXTURE_TEST_CASE(SRA_Reference_Open, NGS_C_Fixture)
{
ENTRY;
@@ -125,7 +704,6 @@ FIXTURE_TEST_CASE(EBI_Reference_Open_EBI_ACC, NGS_C_Fixture)
EXIT;
}
-
//////////////////////////////////////////// Main
extern "C"
{
@@ -146,7 +724,7 @@ rc_t CC Usage ( const Args * args )
return 0;
}
-const char UsageDefaultName[] = "test-ngs_sra";
+const char UsageDefaultName[] = "test-ngs_reference";
rc_t CC KMain ( int argc, char *argv [] )
{
diff --git a/test/ngs/ngstest_referenceblob.cpp b/test/ngs/ngstest_referenceblob.cpp
new file mode 100644
index 0000000..ea6e55e
--- /dev/null
+++ b/test/ngs/ngstest_referenceblob.cpp
@@ -0,0 +1,773 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Unit tests for low-level NGS functions handling blob-bases access to REFERENCE table
+*/
+
+// suppress macro max from windows.h
+#define NOMINMAX
+
+#include "ngs_c_fixture.hpp"
+
+#include <vdb/database.h>
+
+#include <NGS_Cursor.h>
+#include <NGS_ReferenceBlob.h>
+#include <NGS_ReferenceBlobIterator.h>
+
+#include <../libs/ngs/CSRA1_Reference.h>
+#include <../libs/ngs/VByteBlob.h>
+
+#include <stdexcept>
+
+using namespace std;
+using namespace ncbi::NK;
+
+TEST_SUITE(NgsReferenceBlobTestSuite);
+
+const char* CSRA1_Accession = "SRR1063272";
+const char* CSRA1_Accession_WithRepeats = "SRR600094";
+const char* CSRA1_Accession_WithCircular = "SRR1769246";
+
+//////////////////////////////////////////// NGS_ReferenceBlob
+
+class ReferenceBlobFixture : public NGS_C_Fixture
+{
+public:
+ ReferenceBlobFixture ()
+ : m_curs ( 0 ),
+ m_blob ( 0 )
+ {
+ }
+
+ virtual void Release()
+ {
+ if (m_ctx != 0)
+ {
+ if ( m_blob != 0 )
+ {
+ NGS_ReferenceBlobRelease ( m_blob, m_ctx );
+ }
+ if ( m_curs != 0 )
+ {
+ NGS_CursorRelease ( m_curs, m_ctx );
+ }
+ }
+ NGS_C_Fixture :: Release ();
+ }
+
+ void GetCursor(const char* p_acc )
+ {
+ const VDatabase* db;
+ THROW_ON_RC ( VDBManagerOpenDBRead ( m_ctx -> rsrc -> vdb, & db, NULL, p_acc ) );
+ NGS_String* run_name = NGS_StringMake ( m_ctx, "", 0);
+
+ if ( m_curs != 0 )
+ {
+ NGS_CursorRelease ( m_curs, m_ctx );
+ }
+ m_curs = NGS_CursorMakeDb ( m_ctx, db, run_name, "REFERENCE", reference_col_specs, reference_NUM_COLS );
+
+ NGS_StringRelease ( run_name, m_ctx );
+ THROW_ON_RC ( VDatabaseRelease ( db ) );
+ }
+ void GetBlob ( const char* p_acc, int64_t p_rowId, int64_t p_refFirst = 1, int64_t p_refLast = 0 )
+ {
+ GetCursor ( p_acc );
+ if ( m_blob != 0 )
+ {
+ NGS_ReferenceBlobRelease ( m_blob, m_ctx );
+ }
+ m_blob = NGS_ReferenceBlobMake ( m_ctx, m_curs, p_rowId, p_refFirst, p_refLast );
+ }
+
+ void CheckRange( ctx_t ctx, int64_t p_first, uint64_t p_count )
+ {
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+ int64_t first;
+ uint64_t count;
+ ON_FAIL ( NGS_ReferenceBlobRowRange ( m_blob, ctx, & first, & count ) )
+ {
+ throw std :: logic_error ( "NGS_ReferenceBlobRowRange() failed" );
+ }
+ if ( p_first != first || p_count != count )
+ {
+ ostringstream str;
+ str << "CheckRange(first=" << p_first << ", count=" << p_count << ") : first=" << first << ", count=" << count;
+ throw std :: logic_error ( str.str() );
+ }
+ }
+
+ const NGS_Cursor* m_curs;
+ struct NGS_ReferenceBlob* m_blob;
+};
+
+// Make
+
+TEST_CASE ( NGS_ReferenceBlob_Make_BadCursor)
+{
+ HYBRID_FUNC_ENTRY ( rcSRA, rcRow, rcAccessing );
+
+ struct NGS_ReferenceBlob * blob = NGS_ReferenceBlobMake ( ctx, NULL, 1, 1, 0 );
+ REQUIRE_FAILED ();
+ REQUIRE_NULL ( blob );
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Make_BadRowId, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetCursor ( CSRA1_Accession );
+
+ struct NGS_ReferenceBlob * blob = NGS_ReferenceBlobMake ( ctx, m_curs, 1, 2, 0 ); // rowId < refStart
+ REQUIRE_FAILED ();
+ REQUIRE_NULL ( blob );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Make_BadStartId, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetCursor ( CSRA1_Accession );
+
+ struct NGS_ReferenceBlob * blob = NGS_ReferenceBlobMake ( ctx, m_curs, 1, -1, 1 ); // bad refStart
+ REQUIRE_FAILED ();
+ REQUIRE_NULL ( blob );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Make, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetCursor ( CSRA1_Accession );
+
+ m_blob = NGS_ReferenceBlobMake ( ctx, m_curs, 1, 1, 0 );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( m_blob );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_DuplicateRelease, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ // Duplicate
+ NGS_ReferenceBlob* anotherBlob = NGS_ReferenceBlobDuplicate ( m_blob, ctx );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( anotherBlob );
+ // Release
+ NGS_ReferenceBlobRelease ( anotherBlob, ctx );
+
+ EXIT;
+}
+
+// Range
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Range_NullArg_1, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ uint64_t rowCount;
+ NGS_ReferenceBlobRowRange ( m_blob, ctx, NULL, &rowCount );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_EQ ( (uint64_t)4, rowCount);
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Range_NullArg_2, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ int64_t firstRow;
+ NGS_ReferenceBlobRowRange ( m_blob, ctx, &firstRow, NULL );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_EQ ( (int64_t)1, firstRow);
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Range_NullArgs, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ NGS_ReferenceBlobRowRange ( m_blob, ctx, NULL, NULL );
+ REQUIRE ( ! FAILED () );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Range, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+ CheckRange ( ctx, 1, 4 );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_RangeLimited, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1, 1, 3 );
+ CheckRange ( ctx, 1, 3 );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Range_FirstRowInsideBlob, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 3 );
+ CheckRange ( ctx, 3, 1 );
+ EXIT;
+}
+
+// Data
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Data_BadArg, ReferenceBlobFixture )
+{
+ ENTRY;
+
+ const void* data = NGS_ReferenceBlobData ( NULL, ctx );
+ REQUIRE_FAILED ();
+ REQUIRE_NULL ( data );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Data, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ const void* data = NGS_ReferenceBlobData ( m_blob, ctx );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( data );
+ REQUIRE_EQ ( string ( "GAATTCTAAA" ), string ( (const char*)data, 10 ) );
+
+ EXIT;
+}
+
+// Size
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Size_BadArg, ReferenceBlobFixture )
+{
+ ENTRY;
+
+ NGS_ReferenceBlobSize ( NULL, ctx );
+ REQUIRE_FAILED ();
+
+ EXIT;
+}
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_Size, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ REQUIRE_EQ ( (uint64_t)20000, NGS_ReferenceBlobSize ( m_blob, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_UnpackedSize_NoRepeats, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ REQUIRE_EQ ( (uint64_t)20000, NGS_ReferenceBlobUnpackedSize ( m_blob, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_UnpackedSize_WithRepeats, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession_WithRepeats, 96 ); /* this blob consists of 9 repeated all-N rows */
+
+ REQUIRE_EQ ( (uint64_t)45000, NGS_ReferenceBlobUnpackedSize ( m_blob, ctx ) );
+
+ EXIT;
+}
+
+// ResolveOffset
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_ResolveOffset_BadSelf, ReferenceBlobFixture )
+{
+ ENTRY;
+
+ uint64_t inRef;
+ NGS_ReferenceBlobResolveOffset ( NULL, ctx, 0, & inRef, NULL, NULL );
+ REQUIRE_FAILED ();
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_ResolveOffset_OffsetOutOfRange, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ uint64_t inRef;
+ NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 20000, & inRef, NULL, NULL );
+ REQUIRE_FAILED ();
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_ResolveOffset_NullOutputParam, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 0, NULL, NULL, NULL );
+ REQUIRE_FAILED ();
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_ResolveOffset_NullOptionalParams, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 );
+
+ uint64_t inRef;
+ NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 0, & inRef, NULL, NULL );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_ResolveOffset_FirstChunk_NoRepeat, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 1 ); /* ref positions 1-20000 in this blob */
+
+ uint64_t inRef;
+ uint32_t repeatCount;
+ uint64_t increment;
+ NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 6123, & inRef, & repeatCount, & increment );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_EQ ( (uint64_t)6123, inRef );
+ REQUIRE_EQ ( (uint32_t)1, repeatCount );
+ REQUIRE_EQ ( (uint64_t)0, increment );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_ResolveOffset_NotTheFirstReference, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 461, 460 ); /* 2nd row for the reference supercont2.1 */
+
+ uint64_t inRef;
+ uint32_t repeatCount;
+ uint64_t increment;
+ NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 99, & inRef, & repeatCount, & increment );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_EQ ( (uint64_t)5099, inRef ); // relative to the reference the blob is in
+ REQUIRE_EQ ( (uint32_t)1, repeatCount );
+ REQUIRE_EQ ( (uint64_t)0, increment );
+
+ EXIT;
+}
+
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_ResolveOffset_NoRepeat, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession, 5 ); /* ref positions 20001-25000 in this blob */
+
+ uint64_t inRef;
+ uint32_t repeatCount;
+ uint64_t increment;
+ NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 123, & inRef, & repeatCount, & increment );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_EQ ( (uint64_t)20123, inRef );
+ REQUIRE_EQ ( (uint32_t)1, repeatCount );
+ REQUIRE_EQ ( (uint64_t)0, increment );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_ResolveOffset_WithRepeat, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession_WithRepeats, 1 );
+
+ uint64_t inRef;
+ uint32_t repeatCount;
+ uint64_t increment;
+ NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 0, & inRef, & repeatCount, & increment );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_EQ ( (uint64_t)0, inRef );
+ REQUIRE_EQ ( (uint32_t)2, repeatCount );
+ REQUIRE_EQ ( (uint64_t)5000, increment );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlob_ResolveOffset_PastRepeat, ReferenceBlobFixture )
+{
+ ENTRY;
+ GetBlob ( CSRA1_Accession_WithRepeats, 1 );
+
+ uint64_t inRef;
+ uint32_t repeatCount;
+ uint64_t increment;
+ NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 5100, & inRef, & repeatCount, & increment );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_EQ ( (uint64_t)10100, inRef );
+ REQUIRE_EQ ( (uint32_t)1, repeatCount );
+ REQUIRE_EQ ( (uint64_t)0, increment );
+
+ EXIT;
+}
+
+//////////////////////////////////////////// NGS_ReferenceBlobIterator
+
+class ReferenceBlobIteratorFixture : public ReferenceBlobFixture
+{
+public:
+ ReferenceBlobIteratorFixture ()
+ : m_blobIt ( 0 )
+ {
+ }
+
+ virtual void Release()
+ {
+ if (m_ctx != 0)
+ {
+ if ( m_blobIt != 0 )
+ {
+ NGS_ReferenceBlobIteratorRelease ( m_blobIt, m_ctx );
+ }
+ }
+ ReferenceBlobFixture :: Release ();
+ }
+
+ void GetIterator ( const char* p_acc, int64_t p_rowId, int64_t p_lastRowId )
+ {
+ GetCursor ( p_acc );
+ if ( m_blobIt != 0 )
+ {
+ NGS_ReferenceBlobIteratorRelease ( m_blobIt, m_ctx );
+ }
+ m_blobIt = NGS_ReferenceBlobIteratorMake ( m_ctx, m_curs, 1, p_rowId, p_lastRowId );
+ }
+
+ bool NextBlob( ctx_t ctx )
+ {
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+ if ( m_blob != 0)
+ {
+ NGS_ReferenceBlobRelease ( m_blob, m_ctx );
+ m_blob = 0;
+ }
+ TRY ( bool ret = NGS_ReferenceBlobIteratorHasMore ( m_blobIt, ctx ) )
+ {
+ if ( ret )
+ {
+ ON_FAIL ( m_blob = NGS_ReferenceBlobIteratorNext ( m_blobIt, ctx ) )
+ {
+ throw std :: logic_error ( "NGS_ReferenceBlobIteratorNext() failed" );
+ }
+ }
+ return ret;
+ }
+ throw std :: logic_error ( "NGS_ReferenceBlobIteratorHasMore() failed" );
+ }
+
+ struct NGS_ReferenceBlobIterator* m_blobIt;
+};
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIteratorMake_NullCursor, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+
+ struct NGS_ReferenceBlobIterator* blobIt = NGS_ReferenceBlobIteratorMake ( ctx, NULL, 1, 1, 2 );
+ REQUIRE_FAILED ();
+ REQUIRE_NULL ( blobIt );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_CreateRelease, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetCursor ( CSRA1_Accession );
+
+ struct NGS_ReferenceBlobIterator* blobIt = NGS_ReferenceBlobIteratorMake ( ctx, m_curs, 1, 1, 2 );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( blobIt );
+ NGS_ReferenceBlobIteratorRelease ( blobIt, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_DuplicateRelease, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 2 );
+
+ // Duplicate
+ struct NGS_ReferenceBlobIterator* anotherBlobIt = NGS_ReferenceBlobIteratorDuplicate ( m_blobIt, ctx );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( anotherBlobIt );
+ // Release
+ NGS_ReferenceBlobIteratorRelease ( anotherBlobIt, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_Next_BadSelf, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 2 );
+ m_blob = NGS_ReferenceBlobIteratorNext ( NULL, ctx );
+ REQUIRE_FAILED ();
+ REQUIRE_NULL ( m_blob );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_Next_Empty, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 0 );
+
+ m_blob = NGS_ReferenceBlobIteratorNext ( m_blobIt, ctx );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NULL ( m_blob );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_Next, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 10 );
+ REQUIRE ( NextBlob ( ctx ) );
+ CheckRange ( ctx, 1, 4 );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_Limited_Next, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 2 );
+ REQUIRE ( NextBlob ( ctx ) );
+ CheckRange ( ctx, 1, 2 );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_Next_Multiple, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 100 );
+ REQUIRE ( NextBlob ( ctx ) );
+ REQUIRE ( NextBlob ( ctx ) );
+ CheckRange ( ctx, 5, 4 );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_Next_End, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 1 );
+ REQUIRE ( NextBlob ( ctx ) );
+ REQUIRE ( ! NextBlob ( ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_HasMore_Empty, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 0 );
+
+ REQUIRE ( ! NGS_ReferenceBlobIteratorHasMore ( m_blobIt, ctx ) );
+ REQUIRE ( ! FAILED () );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_HasMore, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 1 );
+
+ REQUIRE ( NGS_ReferenceBlobIteratorHasMore ( m_blobIt, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_FullScan, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 1, 459 );
+ size_t count = 0;
+ while ( NGS_ReferenceBlobIteratorHasMore ( m_blobIt, ctx ) )
+ {
+ REQUIRE ( NextBlob ( ctx ) );
+ ++count;
+ }
+ REQUIRE_EQ( (size_t)12, count);
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_ReferenceBlobIterator_MidTable, ReferenceBlobIteratorFixture )
+{
+ ENTRY;
+ GetIterator ( CSRA1_Accession, 100, 106 ); // start in the middle of the reference table
+
+ // move to the second blob in the set
+ REQUIRE ( NextBlob ( ctx ) );
+ REQUIRE ( NextBlob ( ctx ) );
+
+ CheckRange ( ctx, 101, 4 );
+
+ // verify that the blob was given the right reference start row
+ uint64_t inRef;
+ uint32_t repeatCount;
+ uint64_t increment;
+ NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 99, & inRef, & repeatCount, & increment );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_EQ ( (uint64_t)500099, inRef ); // relative to the ref start, which is row 1 in this case
+ REQUIRE_EQ ( (uint32_t)1, repeatCount );
+ REQUIRE_EQ ( (uint64_t)0, increment );
+
+ EXIT;
+}
+
+// NGS_ReferenceGetBlobs
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBlobs_All, ReferenceBlobIteratorFixture)
+{
+ ENTRY_GET_REF ( CSRA1_Accession_WithCircular, "NC_012920.1" );
+
+ m_blobIt = NGS_ReferenceGetBlobs( m_ref, ctx, 0, (uint64_t)-1 );
+ REQUIRE ( ! FAILED () && m_blobIt );
+
+ REQUIRE ( NextBlob ( ctx ) );
+ CheckRange ( ctx, 619147, 1 );
+ REQUIRE ( NextBlob ( ctx ) );
+ CheckRange ( ctx, 619148, 1 );
+ REQUIRE ( NextBlob ( ctx ) );
+ // the actual blob contains 4 rows but only the first 2 are from this reference
+ CheckRange ( ctx, 619149, 2 );
+ // size includes only the 2 reported rows
+ REQUIRE_EQ ( (uint64_t)6569, NGS_ReferenceBlobSize ( m_blob, ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBlobs_Slice_SingleBlob, ReferenceBlobIteratorFixture)
+{
+ ENTRY_GET_REF ( CSRA1_Accession, "supercont2.1" );
+
+ m_blobIt = NGS_ReferenceGetBlobs( m_ref, ctx, 25000, 5000 );
+ REQUIRE ( ! FAILED () && m_blobIt );
+
+ REQUIRE ( NextBlob ( ctx ) );
+ CheckRange ( ctx, 6, 1 );
+ // inside a slice-bound iterator, blobs resolve offsets (inReference) to be in bases from the start of the reference, not the slice
+ uint64_t inReference = 0;
+ TRY (NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 10, &inReference, NULL, NULL ) )
+ {
+ REQUIRE_EQ ( (uint64_t)25010, inReference );
+ }
+
+ REQUIRE ( ! NextBlob ( ctx ) );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBlobs_Slice_MultipleBlobs, ReferenceBlobIteratorFixture)
+{
+ ENTRY_GET_REF ( CSRA1_Accession, "supercont2.1" );
+
+ m_blobIt = NGS_ReferenceGetBlobs( m_ref, ctx, 6000, 5000 );
+ REQUIRE ( ! FAILED () && m_blobIt );
+
+ REQUIRE ( NextBlob ( ctx ) );
+ CheckRange ( ctx, 2, 1 );
+ // inside a slice-bound iterator, blobs resolve offsets (inReference) to be in bases from the start of the reference, not the slice
+ uint64_t inReference = 0;
+ TRY (NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 100, &inReference, NULL, NULL ) )
+ {
+ REQUIRE_EQ ( (uint64_t)5100, inReference );
+ }
+
+ REQUIRE ( NextBlob ( ctx ) );
+ CheckRange ( ctx, 3, 1 );
+ TRY (NGS_ReferenceBlobResolveOffset ( m_blob, ctx, 200, &inReference, NULL, NULL ) )
+ {
+ REQUIRE_EQ ( (uint64_t)10200, inReference );
+ }
+ REQUIRE ( ! NextBlob ( ctx ) );
+
+ EXIT;
+}
+
+//////////////////////////////////////////// Main
+
+extern "C"
+{
+
+#include <kapp/args.h>
+
+ver_t CC KAppVersion ( void )
+{
+ return 0x1000000;
+}
+rc_t CC UsageSummary (const char * progname)
+{
+ return 0;
+}
+
+rc_t CC Usage ( const Args * args )
+{
+ return 0;
+}
+
+const char UsageDefaultName[] = "test-ngs_referenceblob";
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ rc_t m_coll=NgsReferenceBlobTestSuite(argc, argv);
+ return m_coll;
+}
+
+}
+
diff --git a/test/vdb/Makefile b/test/vdb/Makefile
index a2c2848..3ec2d71 100644
--- a/test/vdb/Makefile
+++ b/test/vdb/Makefile
@@ -37,7 +37,7 @@ TEST_TOOLS = \
test-blob-val \
test-VDB-3060 \
test-VDB-3061
-
+
include $(TOP)/build/Makefile.env
@@ -244,3 +244,23 @@ $(TEST_BINDIR)/test-VDB-3061: $(VDB_3061_OBJ)
VDB-3061: test-VDB-3061
$(TEST_BINDIR)/test-VDB-3061
+
+#-------------------------------------------------------------------------------
+# VDB-3305
+#
+VDB_3305_SRC = \
+ test-VDB-3305
+
+VDB_3305_OBJ = \
+ $(addsuffix .$(OBJX),$(VDB_3305_SRC))
+
+VDB_3305_LIB = \
+ -skapp \
+ -sktst \
+ -sncbi-vdb
+
+$(TEST_BINDIR)/test-VDB-3305: $(VDB_3305_OBJ)
+ $(LP) --exe -o $@ $^ $(VDB_3305_LIB)
+
+VDB-3305: test-VDB-3305
+ $(TEST_BINDIR)/test-VDB-3305
diff --git a/test/vdb/test-VDB-3060.cpp b/test/vdb/test-VDB-3060.cpp
index 67a5e4b..4fb1e7d 100644
--- a/test/vdb/test-VDB-3060.cpp
+++ b/test/vdb/test-VDB-3060.cpp
@@ -109,7 +109,7 @@ const char other_path[] = "/some/other/path";
/*
test VDBManagerSetCacheRoot() with invalid and valid parameters
- set the value to "/home/raetzw/somepath"
+ set the value to "/home/user/somepath"
*/
TEST_CASE( SetCacheRoot_1 )
{
diff --git a/test/vdb/test-VDB-3061.cpp b/test/vdb/test-VDB-3061.cpp
index 05dcfce..1ea87ea 100644
--- a/test/vdb/test-VDB-3061.cpp
+++ b/test/vdb/test-VDB-3061.cpp
@@ -44,6 +44,7 @@ using namespace std;
TEST_SUITE( VDB_3061 )
+const char HomeSub[] = "test_root_history";
char new_home[ 4096 ];
static rc_t create_cache_file( KDirectory * dir, const char * path, const char * sub, const char * name, int64_t age )
@@ -77,8 +78,26 @@ static rc_t create_repo_dirs( KDirectory * dir, const char * path, const char *
}
-TEST_CASE( CLEAR_CACHE )
+static void clear_out( const char * path )
{
+ /* clear the temp. home-directory */
+ KDirectory * dir;
+ rc_t rc = KDirectoryNativeDir( &dir );
+ if ( rc == 0 )
+ {
+#ifdef WINDOWS
+ rc = KDirectoryRemove( dir, true, "%s", path );
+#else
+ rc = KDirectoryRemove( dir, true, "%s", path );
+#endif
+ KDirectoryRelease( dir );
+ }
+}
+
+TEST_CASE( CLEAR_CACHE_1 )
+{
+ REQUIRE_RC( KOutMsg( "running: CLEAR_CACHE_1\n" ) );
+
/* create a repository-structure equivalent to the config-values, with 3 files in it */
KDirectory * dir;
REQUIRE_RC( KDirectoryNativeDir( &dir ) );
@@ -102,8 +121,27 @@ TEST_CASE( CLEAR_CACHE )
uint32_t pt3 = KDirectoryPathType( dir, "%s/ncbi/dbGaP-4831/sra/file1.txt", new_home );
REQUIRE_EQ( pt3, (uint32_t)kptFile );
REQUIRE_RC( KDirectoryRelease ( dir ) );
+
+ REQUIRE_RC( KOutMsg( "done: CLEAR_CACHE_1\n" ) );
}
+
+TEST_CASE( CLEAR_CACHE_2 )
+{
+ REQUIRE_RC( KOutMsg( "running: CLEAR_CACHE_2\n" ) );
+
+ REQUIRE_RC( KOutMsg( "clearing: %s\n", new_home ) );
+ clear_out( new_home );
+
+ const VDBManager * vdb_mgr;
+ REQUIRE_RC( VDBManagerMakeRead( &vdb_mgr, NULL ) );
+ REQUIRE_RC( VDBManagerDeleteCacheOlderThan ( vdb_mgr, 0 ) );
+ REQUIRE_RC( VDBManagerRelease ( vdb_mgr ) );
+
+ REQUIRE_RC( KOutMsg( "done: CLEAR_CACHE_2\n" ) );
+}
+
+
static rc_t write_root( KConfig *cfg, const char * base, const char * cat, const char * sub_cat )
{
char key[ 256 ];
diff --git a/test/vdb/test-VDB-3305.cpp b/test/vdb/test-VDB-3305.cpp
new file mode 100644
index 0000000..29cb974
--- /dev/null
+++ b/test/vdb/test-VDB-3305.cpp
@@ -0,0 +1,161 @@
+// ===========================================================================
+//
+// PUBLIC DOMAIN NOTICE
+// National Center for Biotechnology Information
+//
+// This software/database is a "United States Government Work" under the
+// terms of the United States Copyright Act. It was written as part of
+// the author's official duties as a United States Government employee and
+// thus cannot be copyrighted. This software/database is freely available
+// to the public for use. The National Library of Medicine and the U.S.
+// Government have not placed any restriction on its use or reproduction.
+//
+// Although all reasonable efforts have been taken to ensure the accuracy
+// and reliability of the software and data, the NLM and the U.S.
+// Government do not and cannot warrant the performance or results that
+// may be obtained by using this software or data. The NLM and the U.S.
+// Government disclaim all warranties, express or implied, including
+// warranties of performance, merchantability or fitness for any particular
+// purpose.
+//
+// Please cite the author in any work or product based on this material.
+//
+// ===========================================================================
+
+#include <vdb/manager.h> // VDBManager
+#include <kdb/manager.h> // KDBManager
+#include <kdb/kdb-priv.h>
+#include <vdb/vdb-priv.h>
+
+#include <ktst/unit_test.hpp> // TEST_CASE
+#include <vfs/path.h>
+#include <vfs/manager.h>
+#include <klib/text.h>
+#include <klib/out.h>
+#include <klib/printf.h>
+#include <kfs/directory.h>
+#include <kfg/config.h>
+
+#include <sysalloc.h>
+#include <cstdlib>
+#include <stdexcept>
+
+using namespace std;
+
+TEST_SUITE( VDB_3305 )
+
+const VDBManager * vdb_mgr = NULL;
+VFSManager * vfs_mgr = NULL;
+
+static rc_t make_global_managers( void )
+{
+ rc_t rc = VDBManagerMakeRead( &vdb_mgr, NULL );
+ if ( rc != 0 )
+ std::cout << "VDB-3060.VdbFixture: VDBManagerMakeRead() failed" << std::endl;
+ else
+ {
+ rc = VFSManagerMake ( &vfs_mgr );
+ if ( rc != 0 )
+ std::cout << "VdbFixture: VFSManagerMake() failed" << std::endl;
+ }
+ return rc;
+}
+
+static void release_global_managers( void )
+{
+ VFSManagerRelease ( vfs_mgr );
+ VDBManagerRelease ( vdb_mgr );
+}
+
+const char other_path[] = "/some/other/path";
+
+/*
+ test VDBManagerSetCacheRoot() with invalid and valid parameters
+ set the value to "/home/user/somepath"
+*/
+TEST_CASE( SetCacheRoot_1 )
+{
+ VPath * vpath;
+ rc_t rc = VFSManagerMakePath( vfs_mgr, &vpath, other_path );
+ if ( rc != 0 )
+ FAIL( "FAIL: VFSManagerMakePath() failed" );
+
+ rc = VDBManagerSetCacheRoot( vdb_mgr, vpath );
+ if ( rc != 0 )
+ FAIL( "FAIL: VDBManagerSetCacheRoot( mgr, vpath ) failed" );
+
+ if ( vpath != NULL )
+ VPathRelease( vpath );
+}
+
+
+char * org_home;
+const char HomeSub[] = "test_root_history";
+char new_home[ 1024 ];
+char new_home_buffer[ 1024 ]; /* buffer for putenv has to stay alive! */
+
+rc_t prepare_test( const char * sub )
+{
+ org_home = getenv( "HOME" );
+ size_t num_writ;
+ rc_t rc = string_printf ( new_home, sizeof new_home, &num_writ, "%s/%s", org_home, sub );
+ if ( rc == 0 )
+ rc = string_printf ( new_home_buffer, sizeof new_home_buffer, &num_writ, "HOME=%s", new_home );
+ if ( rc == 0 )
+ rc = putenv( new_home_buffer );
+ if ( rc == 0 )
+ {
+ /* create a '.ncbi' directory below */
+ KDirectory * dir;
+ rc = KDirectoryNativeDir( &dir );
+ if ( rc == 0 )
+ {
+ KOutMsg( "%s\n", new_home );
+
+ rc = KDirectoryCreateDir( dir, 0775, ( kcmInit | kcmParents ) , "%s/.ncbi", new_home );
+ KDirectoryRelease( dir );
+ }
+ }
+ return rc;
+}
+
+void finish_test( void )
+{
+ /* clear the temp. home-directory */
+ KDirectory * dir;
+ rc_t rc = KDirectoryNativeDir( &dir );
+ if ( rc == 0 )
+ {
+ rc = KDirectoryRemove( dir, true, "%s/%s", org_home, HomeSub );
+ KDirectoryRelease( dir );
+ }
+}
+
+//////////////////////////////////////////// Main
+extern "C"
+{
+
+#include <kapp/args.h>
+
+ver_t CC KAppVersion ( void ) { return 0x1000000; }
+rc_t CC UsageSummary ( const char * progname ) { return 0; }
+rc_t CC Usage ( const Args * args ) { return 0; }
+const char UsageDefaultName[] = "test-VDB-3305";
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ rc_t rc = prepare_test( HomeSub );
+ if ( rc == 0 )
+ {
+ rc = make_global_managers();
+ if ( rc == 0 )
+ {
+ rc = VDB_3305( argc, argv );
+ release_global_managers();
+ }
+ }
+ //finish_test();
+ return rc;
+}
+
+}
diff --git a/test/vdb/test-vdb.cpp b/test/vdb/test-vdb.cpp
index ade9640..149c8de 100644
--- a/test/vdb/test-vdb.cpp
+++ b/test/vdb/test-vdb.cpp
@@ -197,7 +197,7 @@ class VdbFixture
{
public:
VdbFixture()
- : mgr(0), curs(0), col_idx(~0)
+ : mgr(0), curs(0)
{
if ( VDBManagerMakeRead(&mgr, NULL) != 0 )
throw logic_error ( "VdbFixture: VDBManagerMakeRead failed" );
@@ -211,7 +211,7 @@ public:
throw logic_error ( "~VdbFixture: VCursorRelease failed" );
}
- rc_t Setup( const char * acc, const char* column = "READ_LEN" )
+ rc_t Setup( const char * acc, const char* column[] )
{
const VDatabase *db = NULL;
rc_t rc = VDBManagerOpenDBRead ( mgr, &db, NULL, acc );
@@ -242,16 +242,22 @@ public:
}
}
- if ( rc == 0 )
+ if ( rc == 0 && column[0] != NULL )
{
rc = VTableCreateCursorRead(tbl, &curs);
if ( rc == 0 )
{
- col_idx = ~0;
- rc = VCursorAddColumn ( curs, &col_idx, column );
- if ( rc == 0 )
- {
- rc = VCursorOpen(curs);
+ int i;
+ int const N = (int)(sizeof(col_idx)/sizeof(col_idx[0]));
+ for (i = 0; i < N; ++i)
+ col_idx[i] = (uint32_t)(~0);
+ for (i = 0; column[i] != NULL && rc == 0; ++i) {
+ assert(i < N);
+ rc = VCursorAddColumn ( curs, col_idx + i, column[i] );
+ if ( rc == 0 )
+ {
+ rc = VCursorOpen(curs);
+ }
}
}
}
@@ -276,7 +282,7 @@ public:
THROW_ON_RC ( VCursorOpenRow ( curs ) );
struct VBlob const *blob;
- THROW_ON_RC ( VCursorGetBlob ( curs, &blob, col_idx ) );
+ THROW_ON_RC ( VCursorGetBlob ( curs, &blob, col_idx[0] ) );
bool ret = true;
@@ -302,51 +308,57 @@ public:
const VDBManager * mgr;
const VCursor * curs;
- uint32_t col_idx;
+ uint32_t col_idx[20];
};
FIXTURE_TEST_CASE(TestCursorIsStatic_SingleRowRun1, VdbFixture)
{
- REQUIRE_RC ( Setup ( "SRR002749", "READ_LEN" ) );
+ static char const *columns[] = { "READ_LEN", 0 };
+ REQUIRE_RC ( Setup ( "SRR002749", columns ) );
bool is_static = false;
- REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx, &is_static) );
+ REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx[0], &is_static) );
REQUIRE ( is_static );
}
FIXTURE_TEST_CASE(TestCursorIsStatic_VariableREAD_LEN, VdbFixture)
{
- REQUIRE_RC ( Setup ( "SRR050566", "READ_LEN" ) );
+ static char const *columns[] = { "READ_LEN", 0 };
+ REQUIRE_RC ( Setup ( "SRR050566", columns ) );
bool is_static = true;
- REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx, &is_static) );
+ REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx[0], &is_static) );
REQUIRE ( ! is_static );
}
#if 0
FIXTURE_TEST_CASE(TestCursorIsStatic_SingleRowRun2, VdbFixture)
{
- REQUIRE_RC ( Setup ( "SRR053325", "READ_LEN" ) );
+ static char const *columns[] = { "READ_LEN", 0 };
+ REQUIRE_RC ( Setup ( "SRR053325", columns ) );
bool is_static = false;
- REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx, &is_static) );
+ REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx[0], &is_static) );
REQUIRE ( is_static );
}
#endif
FIXTURE_TEST_CASE(TestCursorIsStatic_FixedREAD_LEN_MultipleRows, VdbFixture)
{
- REQUIRE_RC ( Setup ( "SRR125365", "READ_LEN" ) );
+ static char const *columns[] = { "READ_LEN", 0 };
+ REQUIRE_RC ( Setup ( "SRR125365", columns ) );
bool is_static = false;
- REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx, &is_static) );
+ REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx[0], &is_static) );
REQUIRE ( is_static );
}
FIXTURE_TEST_CASE(TestCursorIsStatic_DB_FixedREAD_LEN_MultipleRows, VdbFixture)
{
- REQUIRE_RC ( Setup ( "SRR600096", "READ_LEN" ) );
+ static char const *columns[] = { "READ_LEN", 0 };
+ REQUIRE_RC ( Setup ( "SRR600096", columns ) );
bool is_static = false;
- REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx, &is_static) );
+ REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx[0], &is_static) );
REQUIRE ( is_static );
}
FIXTURE_TEST_CASE(TestCursorIsStatic_DB_VariableREAD_LEN_MultipleRows, VdbFixture)
{
- REQUIRE_RC ( Setup ( "SRR619505", "READ_LEN" ) );
+ static char const *columns[] = { "READ_LEN", 0 };
+ REQUIRE_RC ( Setup ( "SRR619505", columns ) );
bool is_static = true;
- REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx, &is_static) );
+ REQUIRE_RC ( VCursorIsStaticColumn ( curs, col_idx[0], &is_static) );
REQUIRE ( ! is_static );
}
@@ -354,7 +366,8 @@ FIXTURE_TEST_CASE(TestCursorIsStatic_DB_VariableREAD_LEN_MultipleRows, VdbFixtur
FIXTURE_TEST_CASE(VCursor_GetBlob_SRA, VdbFixture)
{ // multiple fragments per row (some are technical), multiple rows per blob
- REQUIRE_RC ( Setup ( "SRR000123", "READ" ) );
+ static char const *columns[] = { "READ", 0 };
+ REQUIRE_RC ( Setup ( "SRR000123", columns ) );
REQUIRE_RC ( VCursorOpen (curs ) );
{
@@ -362,7 +375,7 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_SRA, VdbFixture)
REQUIRE_RC ( VCursorOpenRow (curs ) );
struct VBlob const *blob;
- REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx ) );
+ REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx[0] ) );
int64_t first;
uint64_t count;
@@ -408,7 +421,7 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_SRA, VdbFixture)
int64_t first;
uint64_t count;
- REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx ) );
+ REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx[0] ) );
REQUIRE_RC ( VBlobIdRange ( blob, &first, &count ) );
REQUIRE_EQ ( (int64_t)5, first );
REQUIRE_EQ ( (uint64_t)4, count );
@@ -424,7 +437,8 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_SRA, VdbFixture)
FIXTURE_TEST_CASE(VCursor_GetBlob_WGS, VdbFixture)
{ // single fragment per row, multiple rows per blob
- REQUIRE_RC ( Setup ( "ALWZ01", "READ" ) );
+ static char const *columns[] = { "READ", 0 };
+ REQUIRE_RC ( Setup ( "ALWZ01", columns ) );
REQUIRE_RC ( VCursorOpen (curs ) );
{
@@ -432,7 +446,7 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_WGS, VdbFixture)
REQUIRE_RC ( VCursorOpenRow (curs ) );
struct VBlob const *blob;
- REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx ) );
+ REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx[0] ) );
int64_t first;
uint64_t count;
@@ -478,7 +492,7 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_WGS, VdbFixture)
int64_t first;
uint64_t count;
- REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx ) );
+ REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx[0] ) );
REQUIRE_RC ( VBlobIdRange ( blob, &first, &count ) );
REQUIRE_EQ ( (int64_t)5, first );
REQUIRE_EQ ( (uint64_t)4, count );
@@ -494,7 +508,8 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_WGS, VdbFixture)
FIXTURE_TEST_CASE(VCursor_GetBlob_SequentialAccess, VdbFixture)
{ // VDB-2858: sequential access to blobs broken
- REQUIRE_RC ( Setup ( "ALAI01", "READ" ) );
+ static char const *columns[] = { "READ", 0 };
+ REQUIRE_RC ( Setup ( "ALAI01", columns ) );
REQUIRE_RC ( VCursorOpen (curs ) );
int64_t first;
@@ -509,7 +524,7 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_SequentialAccess, VdbFixture)
REQUIRE_RC ( VCursorOpenRow ( curs ) );
struct VBlob const *blob;
- if ( VCursorGetBlob ( curs, &blob, col_idx ) != 0 )
+ if ( VCursorGetBlob ( curs, &blob, col_idx[0] ) != 0 )
{
break;
}
@@ -526,7 +541,8 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_SequentialAccess, VdbFixture)
FIXTURE_TEST_CASE(VCursor_GetBlob_RandomAccess, VdbFixture)
{
- REQUIRE_RC ( Setup ( "SRR000001", "READ" ) );
+ static char const *columns[] = { "READ", 0 };
+ REQUIRE_RC ( Setup ( "SRR000001", columns ) );
REQUIRE_RC ( VCursorOpen (curs ) );
// when accessing randomly, blob sizes stay very small
@@ -543,7 +559,8 @@ FIXTURE_TEST_CASE(VCursor_GetBlob_RandomAccess, VdbFixture)
FIXTURE_TEST_CASE(PageMapIterator_WGS, VdbFixture)
{ // single fragment per row, multiple rows per blob
- REQUIRE_RC ( Setup ( "ALWZ01", "READ" ) );
+ static char const *columns[] = { "READ", 0 };
+ REQUIRE_RC ( Setup ( "ALWZ01", columns ) );
REQUIRE_RC ( VCursorOpen (curs ) );
{
@@ -551,7 +568,7 @@ FIXTURE_TEST_CASE(PageMapIterator_WGS, VdbFixture)
REQUIRE_RC ( VCursorOpenRow (curs ) );
struct VBlob const *blob;
- REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx ) );
+ REQUIRE_RC ( VCursorGetBlob ( curs, &blob, col_idx[0] ) );
PageMapIterator pmIt;
REQUIRE_RC ( PageMapNewIterator ( (const PageMap*)blob->pm, &pmIt, 0, 4 ) );
@@ -583,12 +600,13 @@ FIXTURE_TEST_CASE(PageMapIterator_WGS, VdbFixture)
FIXTURE_TEST_CASE ( VCursor_FindNextRowIdDirect, VdbFixture )
{
- REQUIRE_RC ( Setup ( "SRR000001", "READ" ) );
+ static char const *columns[] = { "SPOT_ID", "READ", 0 };
+ REQUIRE_RC ( Setup ( "SRR000001", columns ) );
REQUIRE_RC ( VCursorOpen (curs ) );
int64_t next;
- REQUIRE_RC ( VCursorFindNextRowIdDirect ( curs, col_idx, 1, & next ) );
+ REQUIRE_RC ( VCursorFindNextRowIdDirect ( curs, 0, 1, & next ) );
REQUIRE_EQ ( (int64_t)1, next ) ;
- REQUIRE_RC ( VCursorFindNextRowIdDirect ( curs, col_idx, 2, & next ) );
+ REQUIRE_RC ( VCursorFindNextRowIdDirect ( curs, 0, 2, & next ) );
REQUIRE_EQ ( (int64_t)2, next ) ; // VDB-3075: next == 1
}
diff --git a/test/vfs/Makefile b/test/vfs/Makefile
index 9ac91a2..9bbb2b2 100644
--- a/test/vfs/Makefile
+++ b/test/vfs/Makefile
@@ -29,13 +29,13 @@ TOP ?= $(abspath ../..)
MODULE = test/vfs
TEST_TOOLS = \
- redirect-rejected-names-cgi-http-to-https \
- test-caching \
- test-vfsmanager \
- test-path-c \
- test-path \
- test-resolver
-
+ redirect-rejected-names-cgi-http-to-https \
+ test-caching \
+ test-names-30 \
+ test-path \
+ test-path-c \
+ test-resolver \
+ test-vfsmanager \
include $(TOP)/build/Makefile.env
@@ -104,6 +104,18 @@ $(TEST_BINDIR)/test-caching: $(CACHING_OBJ)
$(LP) --exe -o $@ $^ $(PATHTEST_LIB)
#----------------------------------------------------------------
+# test-names-30
+
+N30_SRC = \
+ test-names-30
+
+N30_OBJ = \
+ $(addsuffix .$(OBJX),$(N30_SRC))
+
+$(TEST_BINDIR)/test-names-30: $(N30_OBJ)
+ $(LP) --exe -o $@ $^ $(PATHTEST_LIB)
+
+#----------------------------------------------------------------
# C++ test on VFSManager
#
diff --git a/test/vfs/redirect-rejected-names-cgi-http-to-https.cpp b/test/vfs/redirect-rejected-names-cgi-http-to-https.cpp
index 192fa6e..f0172c9 100644
--- a/test/vfs/redirect-rejected-names-cgi-http-to-https.cpp
+++ b/test/vfs/redirect-rejected-names-cgi-http-to-https.cpp
@@ -66,13 +66,11 @@ typedef enum {
e403,
} EForbidden;
-class Test : protected ncbi :: NK :: TestCase {
- TestCase * _dad;
-public:
+struct Test : protected ncbi :: NK :: SharedTest {
Test ( const std :: string & description, TestCase * dad,
EForbidden forbidden, const char * name, const char * url,
bool fail = false )
- : TestCase ( name ), _dad ( dad )
+ : SharedTest ( dad, name )
{
rc_t rc = 0;
@@ -146,10 +144,6 @@ public:
REQUIRE ( ! rc );
}
- ~Test ( void ) {
- assert( _dad );
- _dad -> ErrorCounterAdd ( GetErrorCounter () );
- }
};
TEST_CASE(TEST) {
diff --git a/test/vfs/resolvertest.cpp b/test/vfs/resolvertest.cpp
index 6c63893..8d03b10 100644
--- a/test/vfs/resolvertest.cpp
+++ b/test/vfs/resolvertest.cpp
@@ -192,11 +192,15 @@ public:
~ResolverFixtureCustomConfig()
{
- if (vfs && VFSManagerRelease(vfs) != 0)
- throw logic_error ( "~ResolverFixtureCustomConfig: VFSManagerRelease failed" );
+ rc_t rc = VFSManagerRelease(vfs);
+ if ( rc != 0 )
+ fprintf ( stderr, "~ResolverFixtureCustomConfig: VFSManagerRelease()=%u\n", rc );
+ vfs = NULL;
- if (resolver && VResolverRelease(resolver) != 0)
- throw logic_error ( "~ResolverFixtureCustomConfig: VResolverRelease failed" );
+ rc = VResolverRelease(resolver);
+ if ( rc != 0 )
+ fprintf ( stderr, "~ResolverFixtureCustomConfig: VResolverRelease()=%u\n", rc );
+ resolver = NULL;
remove(m_configName.c_str());
}
diff --git a/test/vfs/test-caching.cpp b/test/vfs/test-caching.cpp
index 31e5426..788467c 100644
--- a/test/vfs/test-caching.cpp
+++ b/test/vfs/test-caching.cpp
@@ -203,7 +203,8 @@ public:
{
#define WGS "AFVF01"
const char * acc = type == eRefseq ? "KC702199.1" :
- type == eSra ? "SRR053325" : WGS ".1";
+ type == eSra ? "SRR003325" : WGS ".1";
+// type == eSra ? "SRR053325" : WGS ".1";
rc_t rc = 0;
KDirectory * native = NULL;
REQUIRE_RC ( KDirectoryNativeDir ( & native ) );
@@ -610,6 +611,7 @@ extern "C" {
return 0;
}
rc_t CC KMain ( int argc, char * argv [] ) {
+ KConfigDisableUserSettings ();
return CachingSuite ( argc, argv );
}
}
diff --git a/test/vfs/test-names-30.cpp b/test/vfs/test-names-30.cpp
new file mode 100644
index 0000000..1624058
--- /dev/null
+++ b/test/vfs/test-names-30.cpp
@@ -0,0 +1,503 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+
+#include <klib/time.h> /* KTimeMakeTime */
+#include <vfs/services-priv.h> /* KServiceTestNamesExecuteExt */
+#include "../../libs/vfs/services-priv.h" /* KServiceNames3_0StreamTest */
+#include "../../libs/vfs/path-priv.h" /* VPathEqual */
+#include <vfs/services.h> /* KSrvResponse */
+#include <vfs/path.h> /* VPath */
+#include <klib/debug.h> /* KDbgSetString */
+#include <klib/rc.h>
+#include <klib/text.h> /* CONST_STRING */
+#include <ktst/unit_test.hpp> /* KMain */
+
+//#include <cstdio> // printf
+
+using std :: string;
+
+TEST_SUITE ( Names3_0_TestSuite );
+
+class P {
+ String _tick;
+ String _id;
+ size_t _size;
+
+static int getDigit ( char c, rc_t * rc ) {
+ assert ( rc );
+
+ if ( * rc != 0 )
+ return 0;
+
+ c = tolower ( c );
+ if ( ! isdigit ( c ) && c < 'a' && c > 'f' ) {
+ * rc = RC ( rcVFS, rcQuery, rcExecuting, rcItem, rcIncorrect );
+ return 0;
+ }
+
+ if ( isdigit ( c ) )
+ return c - '0';
+
+ return c - 'a' + 10;
+ }
+
+public:
+ P ( const char * tick, const char * id, size_t size ) : _size ( size ) {
+ size_t s = string_measure ( tick, NULL );
+ StringInit ( & _tick, tick, s, s );
+
+ s = string_measure ( id, NULL );
+ StringInit ( & _id, id, s, s );
+ }
+
+ const VPath * make ( const char * path, const string & date = "",
+ const string & md5 = "", KTime_t expiration = 0 )
+ {
+ KTime_t t = 0;
+ if ( date . size () > 0 ) {
+ KTime kt;
+ const KTime * tt = KTimeFromIso8601 ( & kt, date . c_str (),
+ date . size () );
+ if ( tt != NULL )
+ t = KTimeMakeTime ( & kt );
+ }
+
+ uint8_t ud5 [ 16 ];
+ uint8_t * pd5 = NULL;
+ if ( md5 . size () == 32 ) {
+ rc_t rc = 0;
+ for ( int i = 0; i < 16 && rc == 0; ++ i ) {
+ ud5 [ i ] = getDigit ( md5 [ 2 * i ], & rc ) * 16;
+ ud5 [ i ] += getDigit ( md5 [ 2 * i + 1 ], & rc );
+ }
+ pd5 = rc == 0 ? ud5 : NULL;
+ }
+
+ String url;
+ size_t s = string_measure ( path, NULL );
+ StringInit ( & url, path, s, s );
+
+ VPath * p = NULL;
+ rc_t rc = VPathMakeFromUrl ( & p, & url, & _tick, true, & _id, _size, t,
+ pd5, expiration );
+
+ if ( rc == 0 )
+ rc = VPathMarkHighReliability ( p, true );
+
+ if ( rc != 0 )
+ throw rc;
+
+ return p;
+ }
+};
+
+static P Path ( "ticket", " object-id ", 90 );
+static P Path1 ( " ticke1 ", " object-i1 ", 10 );
+
+#if 1
+TEST_CASE ( INCOMPLETE ) {
+ const KSrvResponse * response = NULL;
+
+ REQUIRE_RC_FAIL ( KServiceNames3_0StreamTest ( NULL, & response, 0 ) );
+ REQUIRE_NULL ( response );
+
+ REQUIRE_RC_FAIL ( KServiceNames3_0StreamTest ( "#3.0\n"
+"0|| object-id |90|1930-01-13T13:25:30|0123456789abcdefABCDEF0123456789|ticket|"
+"http://url/$fasp://frl/$https://hsl/$file:///p$s3:p||||"
+"http://vdbcacheUrl/$fasp://fvdbcache/$https://vdbcache/$file:///vdbcache$s3:v|"
+ "1490000000|200| message\n"
+ "1|| object-i1 |10| dat1 | md1 | ticke1 |"
+ "http://ur1/|https://vdbcacheUrl1/| expiratio1 |200| messag1\n"
+ "$1500000000\n", NULL, 0 ) );
+ REQUIRE_NULL ( response );
+}
+
+TEST_CASE ( SINGLE ) {
+ const KSrvResponse * response = NULL;
+
+ // incomplete string
+ REQUIRE_RC_FAIL ( KServiceNames3_0StreamTest ( "#3.0\n"
+ "SRR000001||http://dwnl.ncbi.nlm.nih.gov/srapub/SRR000001||||200|ok",
+ & response, 0 ) );
+ REQUIRE_NULL ( response );
+ REQUIRE_RC ( KSrvResponseRelease (response ) );
+ response = NULL;
+
+ const string date ( "1980-01-13T13:25:30" );
+ const KTime_t exp = 2000000000 ;
+ const string md5 ( "0123456789abcdefABCDEF012345678a" );
+ const VPath * ph = Path . make ( "http://url/" , date, md5, exp );
+ const VPath * vh = Path . make ( "http://vdbcacheUrl/", "", md5 );
+ const VPath * phs= Path . make ( "https://hsl/" , date, md5, exp );
+ const VPath * vhs= Path . make ( "https://vdbcache/" , "" , md5 );
+ const VPath * pf = Path . make ( "fasp://frl/" , date, md5, exp );
+ const VPath * vf = Path . make ( "fasp://fvdbcache/" , "" , md5 );
+ const VPath * pfl= Path . make ( "file:///p" , date, md5, exp );
+ const VPath * vfl= Path . make ( "file:///vdbcache" , "" , md5 );
+ const VPath * p3 = Path . make ( "s3:p" , date, md5, exp );
+ const VPath * v3 = Path . make ( "s3:v" , "" , md5 );
+
+ REQUIRE_RC ( KServiceNames3_0StreamTest ( "#3.0\n"
+"0|| object-id |90|1980-01-13T13:25:30|0123456789abcdefABCDEF012345678a|ticket|"
+"http://url/$fasp://frl/$https://hsl/$file:///p$s3:p||||"
+"http://vdbcacheUrl/$fasp://fvdbcache/$https://vdbcache/$file:///vdbcache$s3:v|"
+ "2000000000|200| message\n"
+ "$1500000000\n", & response, 0 ) );
+ CHECK_NOT_NULL ( response );
+ REQUIRE_EQ ( KSrvResponseLength ( response ), 1u );
+
+ const VPath * path = NULL;
+ const VPath * vdbcache = NULL;
+
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolDefault,
+ & path, & vdbcache, NULL ) );
+ int ne = ~0;
+ REQUIRE_RC ( VPathEqual ( path, ph, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (path ) );
+ path = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( vdbcache, vh, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (vdbcache ) );
+ vdbcache = NULL;
+
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolHttp,
+ & path, & vdbcache, NULL ) );
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( path, ph, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (path ) );
+ path = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( vdbcache, vh, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (vdbcache ) );
+ vdbcache = NULL;
+
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolHttpHttps,
+ & path, & vdbcache, NULL ) );
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( path, ph, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (path ) );
+ path = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( vdbcache, vh, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (vdbcache ) );
+ vdbcache = NULL;
+
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolFaspHttp,
+ & path, & vdbcache, NULL ) );
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( path, pf, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (path ) );
+ path = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( vdbcache, vf, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (vdbcache ) );
+ vdbcache = NULL;
+
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolFileFaspHttpHttps,
+ & path, & vdbcache, NULL ) );
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( path, pfl, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (path ) );
+ path = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( vdbcache, vfl, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (vdbcache ) );
+ vdbcache = NULL;
+
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolHttpsHttp,
+ & path, & vdbcache, NULL ) );
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( path, phs, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (path ) );
+ path = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( vdbcache, vhs, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (vdbcache ) );
+ vdbcache = NULL;
+
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolS3,
+ & path, & vdbcache, NULL ) );
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( path, p3, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (path ) );
+ path = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathEqual ( vdbcache, v3, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (vdbcache ) );
+ vdbcache = NULL;
+
+ REQUIRE_RC ( KSrvResponseRelease (response ) );
+ response = NULL;
+
+ REQUIRE_RC ( VPathRelease ( ph ) );
+ REQUIRE_RC ( VPathRelease ( vh ) );
+ REQUIRE_RC ( VPathRelease ( phs) );
+ REQUIRE_RC ( VPathRelease ( vhs) );
+ REQUIRE_RC ( VPathRelease ( pf ) );
+ REQUIRE_RC ( VPathRelease ( vf ) );
+ REQUIRE_RC ( VPathRelease ( pfl) );
+ REQUIRE_RC ( VPathRelease ( vfl) );
+ REQUIRE_RC ( VPathRelease ( p3 ) );
+ REQUIRE_RC ( VPathRelease ( v3 ) );
+}
+
+TEST_CASE ( DOUBLE ) {
+ const KSrvResponse * response = NULL;
+
+ const string date ( "1981-01-13T13:25:30" );
+ const KTime_t exp = 1489700000 ;
+ const string date1 ( "1981-01-13T13:25:31" );
+ const KTime_t exp1 = 1489710000 ;
+ REQUIRE_RC ( KServiceNames3_0StreamTest ( "#3.0\n"
+ "0|| object-id |90|1981-01-13T13:25:30||ticket|"
+"http://url/$fasp://frl/$https://hsl/$file:///p$s3:p||||"
+"http://vdbcacheUrl/$fasp://fvdbcache/$https://vdbcache/$file:///vdbcache$s3:v|"
+ "1489700000|200| message\n"
+ "1|| object-i1 |10|1981-01-13T13:25:31|| ticke1 |"
+ "http://ur1/||||https://vdbcacheUrl1/|1489710000|200| messag1\n"
+ "$1489690000\n", & response, 0 ) );
+
+ CHECK_NOT_NULL ( response );
+ REQUIRE_EQ ( KSrvResponseLength ( response ), 2u );
+
+ const VPath * phs = Path . make ( "https://hsl/", date, "", exp );
+ const VPath * path = NULL;
+ const VPath * vdbcache = NULL;
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolHttps,
+ & path, NULL, NULL ) );
+ int ne = ~0;
+ REQUIRE_RC ( VPathEqual ( path, phs, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (path ) );
+ path = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathRelease (phs ) );
+
+ const VPath * ph = Path1 . make ( "http://ur1/", date1, "", exp1 );
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 1, eProtocolHttp,
+ & path, & vdbcache, NULL ) );
+ REQUIRE_NULL ( vdbcache );
+ REQUIRE_RC ( VPathEqual ( path, ph, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (path ) );
+ path = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathRelease (ph ) );
+
+ const VPath * vhs = Path1 . make ( "https://vdbcacheUrl1/" );
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 1, eProtocolHttps,
+ & path, & vdbcache, NULL ) );
+ REQUIRE_NULL ( path );
+ REQUIRE_RC ( VPathEqual ( vdbcache, vhs, & ne ) );
+ REQUIRE_EQ ( ne, 0 );
+ REQUIRE_RC ( VPathRelease (vdbcache ) );
+ vdbcache = NULL;
+ ne = ~0;
+ REQUIRE_RC ( VPathRelease (vhs ) );
+
+ REQUIRE_RC ( KSrvResponseRelease (response ) );
+ response = NULL;
+}
+
+TEST_CASE ( BAD_TYPE ) {
+ const KSrvResponse * response = NULL;
+ REQUIRE_RC_FAIL ( KServiceNames3_0StreamTest ( "#3.0\n"
+ "0|t| object-id |9|1981-01-13T13:25:30|0123456789abcdefABCDEF012345678b"
+ "|ticket|||1981-01-13T13:25:30||||200| mssg\n",
+ & response, 1 ) );
+ REQUIRE_NULL ( response );
+}
+
+TEST_CASE ( ERROR ) {
+ const KSrvResponse * response = NULL;
+ REQUIRE_RC ( KServiceNames3_0StreamTest ( "#3.0\n"
+ "0|| object-id |90|1981-01-13T13:25:30|0123456789abcdefABCDEF012345678c"
+ "|ticket||||||1489688000|500| mssg\n",
+ & response, 1 ) );
+ REQUIRE_NOT_NULL ( response );
+ REQUIRE_EQ ( KSrvResponseLength ( response ), 1u );
+
+ const VPath * path = NULL;
+ const VPath * vdbcache = NULL;
+ const KSrvError * error = NULL;
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolDefault,
+ & path, & vdbcache, & error ) );
+ REQUIRE_NULL ( path );
+ REQUIRE_NULL ( vdbcache );
+ REQUIRE_NOT_NULL ( error );
+ REQUIRE_RC ( KSrvErrorRelease ( error ) );
+
+ REQUIRE_RC ( KSrvResponseRelease ( response ) );
+ response = NULL;
+}
+
+TEST_CASE ( AND_ERROR ) {
+ const KSrvResponse * response = NULL;
+ REQUIRE_RC ( KServiceNames3_0StreamTest ( "#3.0\n"
+"0|na|object-0|90|1930-01-13T13:25:30|0123456789abcdefABCDEF012345678d|tckt0|||"
+ "|||1489687900|503|e mssg\n"
+"1||objc1|10|1931-01-13T13:25:31|0123456789abcdefABCDEF012345678e|1|http://u/||"
+ "|||1489687200|200|messag\n"
+ , & response, 1 ) );
+ REQUIRE_NOT_NULL ( response );
+ REQUIRE_EQ ( KSrvResponseLength ( response ), 2u );
+
+ const VPath * path = NULL;
+ const VPath * vdbcache = NULL;
+ const KSrvError * error = NULL;
+
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 0, eProtocolDefault,
+ & path, & vdbcache, & error ) );
+ REQUIRE_NULL ( path );
+ REQUIRE_NULL ( vdbcache );
+ REQUIRE_NOT_NULL ( error );
+ rc_t rc = 0;
+ REQUIRE_RC ( KSrvErrorRc ( error, & rc ) );
+ REQUIRE_EQ ( rc,
+ RC ( rcVFS, rcQuery, rcResolving, rcDatabase, rcNotAvailable ) );
+ uint32_t code = 0;
+ REQUIRE_RC ( KSrvErrorCode ( error, & code ) );
+ REQUIRE_EQ ( code, 503u );
+ String str;
+ REQUIRE_RC ( KSrvErrorMessage ( error, & str ) );
+ String exp;
+ CONST_STRING ( & exp, "e mssg" );
+ REQUIRE ( StringEqual ( & str, & exp ) );
+/* printf ( "KSrvErrorMessage: '%.*s'\n", (int)message. size, message. addr );
+ printf ( "Expected : '%.*s'\n", (int)exp . size, exp . addr );
+ printf ( "KSrvErrorMessage.len: '%d'\n", message. len );
+ printf ( "Expected .len: '%d'\n", exp . len );
+ printf ( "KSrvErrorMessage.sz : '%ld'\n", message. size );
+ printf ( "Expected .sz : '%ld'\n", exp . size );*/
+ EObjectType type = eOT_undefined;
+ REQUIRE_RC ( KSrvErrorObject ( error, & str, & type ) );
+ REQUIRE_EQ ( type, eOT_na );
+ CONST_STRING ( & exp, "object-0" );
+ REQUIRE ( StringEqual ( & str, & exp ) );
+ REQUIRE_RC ( KSrvErrorRelease ( error ) );
+
+ REQUIRE_RC ( KSrvResponseGetPath ( response, 1, eProtocolDefault,
+ & path, & vdbcache, & error ) );
+ REQUIRE_NULL ( vdbcache );
+ REQUIRE_NULL ( error );
+ REQUIRE_NOT_NULL ( path );
+ REQUIRE_RC ( VPathRelease ( path ) );
+
+ REQUIRE_RC ( KSrvResponseRelease ( response ) );
+ response = NULL;
+}
+#endif
+
+TEST_CASE ( FULL_TEST_NO_HTTP ) {
+// assert ( ! KDbgSetString ( "VFS" ) );
+
+ REQUIRE_RC_FAIL ( KServiceMake ( NULL ) );
+
+ KService * service = NULL;
+ REQUIRE_RC ( KServiceMake ( & service ) );
+ REQUIRE_NOT_NULL ( service );
+
+ REQUIRE_RC_FAIL ( KServiceTestNamesExecuteExt ( service, 0, NULL, NULL,
+ NULL, NULL ) );
+
+ const KSrvResponse * response = NULL;
+if ( 1 )
+ REQUIRE_RC_FAIL ( KServiceTestNamesExecuteExt ( service, 0, NULL, NULL,
+ & response, "" ) );
+
+ REQUIRE_RC_FAIL ( KServiceAddId ( NULL, "SRR000001" ) );
+
+ REQUIRE_RC ( KServiceAddId ( service, "SRR000001" ) );
+#if 0
+ REQUIRE_RC_FAIL ( KServiceTestNamesExecuteExt ( service, 0, NULL, "#3.0",
+ & response, "" ) );
+ REQUIRE_RC ( KServiceTestNamesExecuteExt ( service, 0, NULL, "#1.2",
+ & response, NULL ) );
+ REQUIRE_EQ ( KSrvResponseLength ( response ), 1u );
+ REQUIRE_RC ( KSrvResponseRelease ( response ));
+
+ REQUIRE_RC ( KServiceTestNamesExecuteExt ( service, 0, NULL, "#1.2",
+ & response, "#1.2\n"
+ "SRR000001||SRR000001|312527083|2015-04-07T21:54:15|"
+ "9bde35fefa9d955f457e22d9be52bcd9||"
+ "http://sra-download.ncbi.nlm.nih.gov/srapub/SRR000001|200|ok\n" ) );
+ REQUIRE_EQ ( KSrvResponseLength ( response ), 1u );
+ REQUIRE_RC ( KSrvResponseRelease ( response ));
+
+ REQUIRE_RC ( KServiceTestNamesExecuteExt ( service, 0, NULL, NULL,
+ & response, NULL ) );
+ REQUIRE_EQ ( KSrvResponseLength ( response ), 1u );
+ REQUIRE_RC ( KSrvResponseRelease ( response ));
+
+ REQUIRE_RC ( KServiceTestNamesExecuteExt ( service, 0, NULL, NULL,
+ & response, "#3.0\n"
+ "srapub|SRR000001|312527083|2015-04-07T21:54:15|"
+ "9bde35fefa9d955f457e22d9be52bcd9||"
+ "http://sra-download.ncbi.nlm.nih.gov/srapub/SRR000001|200|ok\n" ) );
+ REQUIRE_EQ ( KSrvResponseLength ( response ), 1u );
+ REQUIRE_RC ( KSrvResponseRelease ( response ));
+#endif
+
+ REQUIRE_RC ( KServiceTestNamesExecuteExt ( service, 0, NULL, "#3.0",
+ & response, "#3.0\n"
+ "0|srapub|SRR000001|312527083|2015-04-07T21:54:15|"
+ "9bde35fefa9d955f457e22d9be52bcd9||"
+ "http://sra-download.ncbi.nlm.nih.gov/srapub/SRR000001||||||200|ok\n"
+ ) );
+ REQUIRE_EQ ( KSrvResponseLength ( response ), 1u );
+ REQUIRE_RC ( KSrvResponseRelease ( response ));
+
+ REQUIRE_RC ( KServiceRelease ( service ) );
+
+ REQUIRE_RC_FAIL ( KServiceTestNamesExecuteExt ( service, 0, NULL, NULL,
+ NULL, NULL ) );
+}
+
+TEST_CASE ( TEST_KFG ) {
+}
+
+extern "C" {
+ ver_t CC KAppVersion ( void ) { return 0; }
+ rc_t CC KMain ( int argc, char * argv [] ) {
+ if ( 0 ) assert ( ! KDbgSetString ( "VFS" ) );
+ return Names3_0_TestSuite ( argc, argv );
+ }
+}
diff --git a/vdb3/itf/kfc/except.hpp b/vdb3/itf/kfc/except.hpp
index d43d54b..46c6a0b 100644
--- a/vdb3/itf/kfc/except.hpp
+++ b/vdb3/itf/kfc/except.hpp
@@ -27,13 +27,25 @@
#ifndef _hpp_vdb3_kfc_except_
#define _hpp_vdb3_kfc_except_
-#ifndef _hpp_vdb3_kfc_defs_
-#include <kfc/defs.hpp>
+/*------------------------------------------------------------------
+ * XC_DECLARE
+ * provides a simple means of declaring a new exception type
+ */
+//#ifdef _hpp_vdb3_kfc_string_
+#ifdef __FILE__
+#define XC_DECLARE ( new_xc, parent_xc ) \
+ struct new_xc : parent_xc \
+ { new_xc ( vdb3 :: U32 lineno, const char * msg ) \
+ : parent_xc ( lineno, msg ) {} }
+#else
+#define XC_DECLARE ( new_xc, parent_xc ) \
+ struct new_xc : parent_xc \
+ { new_xc ( vdb3 :: U32 lineno, const vdb3 :: String & msg ) \
+ : parent_xc ( lineno, msg ) {} }
#endif
-#ifndef _hpp_vdb3_kfc_string_
+#include <kfc/defs.hpp>
#include <kfc/string.hpp>
-#endif
namespace vdb3
{
@@ -101,22 +113,6 @@ namespace vdb3
};
- /*------------------------------------------------------------------
- * XC_DECLARE
- * provides a simple means of declaring a new exception type
- */
-#ifdef _hpp_vdb3_kfc_string_
-#define XC_DECLARE( new_xc, parent_xc ) \
- struct new_xc : parent_xc \
- { new_xc ( vdb3 :: U32 lineno, const vdb3 :: String & msg ) \
- : parent_xc ( lineno, msg ) {} }
-#else
-#define XC_DECLARE( new_xc, parent_xc ) \
- struct new_xc : parent_xc \
- { new_xc ( vdb3 :: U32 lineno, const char * msg ) \
- : parent_xc ( lineno, msg ) {} }
-#endif
-
XC_DECLARE ( logic_err, exception );
XC_DECLARE ( runtime_err, exception );
XC_DECLARE ( usage_err, exception );
diff --git a/vdb3/itf/kfc/memmgr.hpp b/vdb3/itf/kfc/memmgr.hpp
index 8f4ae7b..ffa0df8 100644
--- a/vdb3/itf/kfc/memmgr.hpp
+++ b/vdb3/itf/kfc/memmgr.hpp
@@ -27,17 +27,10 @@
#ifndef _hpp_vdb3_kfc_memmgr_
#define _hpp_vdb3_kfc_memmgr_
-#ifndef _hpp_vdb3_kfc_except_
#include <kfc/except.hpp>
-#endif
-
-#ifndef _hpp_vdb3_kfc_refcount_
#include <kfc/refcount.hpp>
-#endif
-
-#ifndef _hpp_vdb3_kfc_ref_
#include <kfc/ref.hpp>
-#endif
+#include <kfc/time.hpp>
namespace vdb3
{
diff --git a/vdb3/itf/kfc/ptr.hpp b/vdb3/itf/kfc/ptr.hpp
index 70f6338..0c8d39c 100644
--- a/vdb3/itf/kfc/ptr.hpp
+++ b/vdb3/itf/kfc/ptr.hpp
@@ -27,13 +27,12 @@
#ifndef _hpp_vdb3_kfc_ptr_
#define _hpp_vdb3_kfc_ptr_
-#ifndef _hpp_vdb3_kfc_memory_
-#include <kfc/memory.hpp>
-#endif
-
-#ifndef _hpp_vdb3_kfc_except_
+/*
#include <kfc/except.hpp>
-#endif
+#include <kfc/memory.hpp>
+#include <kfc/ref.hpp>
+#include <kfc/string.hpp>
+*/
namespace vdb3
{
@@ -41,6 +40,14 @@ namespace vdb3
/*------------------------------------------------------------------
* exceptions
*/
+
+ class new_xc : vdb3::exception
+ {
+ public:
+ new_xc ( vdb3 :: U32 lineno, const vdb3 :: String & msg ) : logic_err( lineno, msg )
+ {}
+ }
+
XC_DECLARE ( xc_ptr_space_err, logic_err );
XC_DECLARE ( xc_ptr_size_err, xc_elem_size_err );
@@ -170,5 +177,4 @@ namespace vdb3
: OpaquePtr ( m, sizeof ( T ) ) {}
};
}
-
#endif // _hpp_vdb3_kfc_ptr_
diff --git a/vdb3/itf/kfc/rsrc.hpp b/vdb3/itf/kfc/rsrc.hpp
index 8396378..12a65f7 100644
--- a/vdb3/itf/kfc/rsrc.hpp
+++ b/vdb3/itf/kfc/rsrc.hpp
@@ -27,25 +27,12 @@
#ifndef _hpp_vdb3_kfc_rsrc_
#define _hpp_vdb3_kfc_rsrc_
-#ifndef _hpp_vdb3_kfc_memmgr_
+#include <kfc/except.hpp>
#include <kfc/memmgr.hpp>
-#endif
-
-#ifndef _hpp_vdb3_kfc_timemgr_
#include <kfc/timemgr.hpp>
-#endif
-
-#ifndef _hpp_vdb3_kfc_fdmgr_
#include <kfc/fdmgr.hpp>
-#endif
-
-#ifndef _hpp_vdb3_kfc_log_
#include <kfc/log.hpp>
-#endif
-
-#ifndef _hpp_vdb3_kfc_stream_
#include <kfc/stream.hpp>
-#endif
namespace vdb3
{
diff --git a/vdb3/itf/kfc/string.hpp b/vdb3/itf/kfc/string.hpp
index ef108ff..ad36adb 100644
--- a/vdb3/itf/kfc/string.hpp
+++ b/vdb3/itf/kfc/string.hpp
@@ -27,9 +27,10 @@
#ifndef _hpp_vdb3_kfc_string_
#define _hpp_vdb3_kfc_string_
-#ifndef _hpp_vdb3_kfc_memory_
+namespace vdb3 { class String; } // forwards for includes
+
#include <kfc/memory.hpp>
-#endif
+#include <kfc/ptr.hpp>
namespace vdb3
{
@@ -155,10 +156,12 @@ namespace vdb3
public:
// supports nasty cast to NUL-terminated string
- operator const char * () const;
+ operator char const * () const;
NULTermString ( const String & str );
void operator = ( const String & str );
+ private:
+ const Ptr < char > rgn;
};
diff --git a/vdb3/src/kfc/Makefile b/vdb3/src/kfc/Makefile
index 44b80b7..0e8ddd4 100644
--- a/vdb3/src/kfc/Makefile
+++ b/vdb3/src/kfc/Makefile
@@ -26,7 +26,7 @@
default: std
TOP ?= $(abspath ../../..)
-include $(TOP)/build/Makefile.config
+MODULE = vdb3/src/kfc
# repairs to Makefile.config
INCDIRS = -I. -I$(TOP)/vdb3/itf
@@ -77,12 +77,20 @@ all std: $(TARGETS)
compile: $(OBJDIR) $(OBJFILES)
clean:
- rm -rf $(ILIBDIR)/$(LPFX)ngs-disp* $(OBJDIR)
+ rm -rf $(ILIBDIR)/$(LPFX)vdb3* $(OBJDIR)
.PHONY: default all std $(TARGETS)
-vdb3-kfc: $(ILIBDIR) $(OBJDIR) $(ILIBDIR)/$(LPFX)vdb3-kfc.$(LIBX)
-
# rule to produce the static library
+vdb3-kfc: $(ILIBDIR) $(OBJDIR) $(ILIBDIR)/$(LPFX)vdb3-kfc.$(LIBX)
$(ILIBDIR)/$(LPFX)vdb3-kfc.$(LIBX): $(KFC_OBJ)
$(AR) $@ $^
+
+libvdb3: $(ILIBDIR) $(OBJDIR) $(ILIBDIR)/$(LPFX)vdb3.$(LIBX)
+$(ILIBDIR)/$(LPFX)vdb3.$(LIBX): $(VDB3_OBJ)
+ $(AR) $@ $^
+
+compile: stdcompile
+
+$(TARGDIR)/compile: $(OBJFILES) libvdb3
+
diff --git a/vdb3/src/kfc/rsrc.cpp b/vdb3/src/kfc/rsrc.cpp
index 08ae36e..966e2e5 100644
--- a/vdb3/src/kfc/rsrc.cpp
+++ b/vdb3/src/kfc/rsrc.cpp
@@ -29,6 +29,7 @@
#include <kfc/except.hpp>
#include <kfc/caps.hpp>
#include <kfc/array.hpp>
+#include <kfc/string.hpp>
#include "pmemmgr.hpp"
#include "ptimemgr.hpp"
#include "pfdmgr.hpp"
diff --git a/vdb3/src/kfc/string.cpp b/vdb3/src/kfc/string.cpp
index 26a2b3e..25138cc 100644
--- a/vdb3/src/kfc/string.cpp
+++ b/vdb3/src/kfc/string.cpp
@@ -1320,5 +1320,83 @@ namespace vdb3
buffer . resize ( rounded_size, true );
}
}
-
+
+ /*
+ * NULTermString
+ * create a string that is NUL-terminated,
+ * and sports a cast to const char *
+ * for use with native OS
+ */
+
+ // supports nasty cast to NUL-terminated string
+ NULTermString :: operator const char * () const
+ {
+ // attempt access to the raw memory
+ const Ptr < char > p = rgn;
+
+ // the memory block must not be empty
+ // or addressing the array will fail
+ if ( len != 0 )
+ {
+ assert ( rgn . size () != ( U64 ) 0 );
+ return & p [ 0 ];
+ }
+
+ // for empty strings, just return an empty C string
+ return "";
+ }
+
+ NULTermString :: NULTermString ( const String & str )
+ : String ( StringBuffer ( "%sz", & str ) . toString () )
+ {
+ // the current value is a copy of input
+ // with an additional character added - space for NUL
+ bytes_t cur_size = rgn . size ();
+ if ( ascii_size == cur_size )
+ -- ascii_size;
+ -- len;
+ -- cur_size;
+
+ // change last character to NUL
+ // access mem as a read-only array
+ // because capabilities have been reduced
+ Ptr < ascii > ptr = rgn;
+ const ascii * p = & ptr [ cur_size ];
+
+ // having stolen a pointer,
+ // use C to hack our own system
+ * ( ascii * ) p = 0;
+
+ // drop size of memory without losing
+ // the guaranteed trailing NUL byte
+ Region r = rgn . subRegion ( 0, cur_size );
+
+ // forget the initial copy
+ rgn = r;
+ }
+
+
+ void NULTermString :: operator = ( const String & str )
+ {
+ // copy the string with an additional character
+ {
+ StringBuffer buf ( "%sz", & str );
+ String :: operator = ( buf . toString () );
+ }
+
+ // see constructor above for comments
+ bytes_t cur_size = rgn . size ();
+ if ( ascii_size == cur_size )
+ -- ascii_size;
+ -- len;
+ -- cur_size;
+
+ Ptr < ascii > a = rgn;
+ const ascii * p = & a [ cur_size ];
+ * ( U8 * ) p = 0;
+
+ Region r = rgn . subRegion ( 0, cur_size );
+ rgn = r;
+ }
+
}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/ncbi-vdb.git
More information about the debian-med-commit
mailing list