[Forensics-changes] [SCM] debian-forensics/libphash branch, upstream, updated. upstream/0.8.1-1-g28baf14
Tiago Bortoletto Vaz
tiago at debian-ba.org
Sun Mar 28 18:16:11 UTC 2010
The following commit has been merged in the upstream branch:
commit 28baf1467a1c82c6310cffd0aac728b3009b3b72
Author: Tiago Bortoletto Vaz <tiago at debian-ba.org>
Date: Sun Mar 28 15:12:46 2010 -0300
Adding upstream version version.
diff --git a/ChangeLog b/ChangeLog
index 3f8bb33..ab443f6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,17 @@
+
+03.27.2010 added multithreaded image, video and audio hash functions to take advantage of
+ multiple cores/cpus.
+03.24.2010 fix heap corruption error in mvp functions. fix ph_readaudio to read ogg and flac
+ audio files in addition to .wav and .mp3.
+ Fix bug in ph_dct_image hash to handle rgba images.
+
+03.18.2010 add example files: build_mvptree_dctimage.cpp , add_mvptree_dct.cpp, query_mvptree_dct.cpp
+
+0.8.2
+---
+02.04.2010 include fix for ffmpeg header location on debian, function prototype change
+
+
0.8.1
---
01.28.2010 minor bug fix for mh image hash and compilation fix for older gcc versions
diff --git a/Makefile.in b/Makefile.in
index e3a6d56..56d0d63 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -41,7 +41,8 @@ DIST_COMMON = README $(am__configure_deps) $(srcdir)/Makefile.am \
ChangeLog INSTALL NEWS TODO config.guess config.sub depcomp \
install-sh ltmain.sh missing
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/configure.ac
+am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \
+ $(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
@@ -186,6 +187,9 @@ PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
+PTHREAD_CC = @PTHREAD_CC@
+PTHREAD_CFLAGS = @PTHREAD_CFLAGS@
+PTHREAD_LIBS = @PTHREAD_LIBS@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
@@ -205,6 +209,7 @@ am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
+ax_pthread_config = @ax_pthread_config@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
diff --git a/bindings/Makefile.in b/bindings/Makefile.in
index 1305c7d..58ef694 100644
--- a/bindings/Makefile.in
+++ b/bindings/Makefile.in
@@ -36,7 +36,8 @@ host_triplet = @host@
subdir = bindings
DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/configure.ac
+am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \
+ $(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
mkinstalldirs = $(install_sh) -d
@@ -146,6 +147,9 @@ PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
+PTHREAD_CC = @PTHREAD_CC@
+PTHREAD_CFLAGS = @PTHREAD_CFLAGS@
+PTHREAD_LIBS = @PTHREAD_LIBS@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
@@ -165,6 +169,7 @@ am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
+ax_pthread_config = @ax_pthread_config@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
diff --git a/bindings/java/MVPTree.java b/bindings/java/MVPTree.java
index e18ce59..5f46cb4 100644
--- a/bindings/java/MVPTree.java
+++ b/bindings/java/MVPTree.java
@@ -3,7 +3,8 @@ public class MVPTree
String mvpFile;
MVPTree(String filename) { mvpFile = filename; }
public native boolean create(Hash[] hashes);
-public native Hash[] query(Hash hash, float radius, int maxResults);
+public native Hash[] query(Hash hash, float radius, float thresh,
+ int maxResults);
public native boolean add(Hash[] hashes);
}
diff --git a/bindings/java/Makefile.in b/bindings/java/Makefile.in
index a0eeb7b..9d06d1c 100644
--- a/bindings/java/Makefile.in
+++ b/bindings/java/Makefile.in
@@ -39,7 +39,8 @@ subdir = bindings/java
DIST_COMMON = README $(include_HEADERS) $(srcdir)/Makefile.am \
$(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/configure.ac
+am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \
+ $(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
mkinstalldirs = $(install_sh) -d
@@ -151,6 +152,9 @@ PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
+PTHREAD_CC = @PTHREAD_CC@
+PTHREAD_CFLAGS = @PTHREAD_CFLAGS@
+PTHREAD_LIBS = @PTHREAD_LIBS@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
@@ -170,6 +174,7 @@ am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
+ax_pthread_config = @ax_pthread_config@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
diff --git a/bindings/java/pHash-jni.cpp b/bindings/java/pHash-jni.cpp
index 747ee84..d56731d 100644
--- a/bindings/java/pHash-jni.cpp
+++ b/bindings/java/pHash-jni.cpp
@@ -136,7 +136,7 @@ JNIEXPORT jboolean JNICALL Java_MVPTree_create
MVPFile mvpfile;
ph_mvp_init(&mvpfile);
- mvpfile.filename = e->GetStringUTFChars(mvp, 0);
+ mvpfile.filename = strdup(e->GetStringUTFChars(mvp, 0));
jniHashType type;
jobject htype = e->GetObjectArrayElement(hashArray, 0);
for(int i = 0; i < sizeof(hashes)/sizeof(hashes[0]); i++)
@@ -156,7 +156,7 @@ JNIEXPORT jboolean JNICALL Java_MVPTree_create
{
jobject hashObj = e->GetObjectArrayElement(hashArray, i);
- hashlist[i] = ph_malloc_datapoint(mvpfile.hash_type, mvpfile.pathlength);
+ hashlist[i] = ph_malloc_datapoint(mvpfile.hash_type);
if(!hashlist[i])
{
free(hashlist);
@@ -216,7 +216,7 @@ JNIEXPORT jboolean JNICALL Java_MVPTree_create
const int sr = 8000;
const int channels = 1;
int nbframes, N;
- float *buf = ph_readaudio(path,sr,channels,N);
+ float *buf = ph_readaudio(path,sr,channels,NULL,N);
if(buf)
{
uint32_t *audioHash = ph_audiohash(buf,N,sr,nbframes);
@@ -247,11 +247,13 @@ JNIEXPORT jboolean JNICALL Java_MVPTree_create
free(hashlist);
e->ReleaseStringUTFChars(mvp, mvpfile.filename);
- return ret == 0 ? JNI_TRUE : JNI_FALSE;
+ free(mvpfile.filename);
+ return (int)ret;
}
JNIEXPORT jobjectArray JNICALL Java_MVPTree_query
- (JNIEnv *e, jobject ob, jobject hashObj, jfloat radius, jint max)
+ (JNIEnv *e, jobject ob, jobject hashObj, jfloat radius,
+ jfloat thresh, jint max)
{
MVPFile mvpfile;
@@ -261,7 +263,7 @@ JNIEXPORT jobjectArray JNICALL Java_MVPTree_query
jstring mvp = (jstring)e->GetObjectField(ob, e->GetFieldID(e->FindClass("MVPTree"), "mvpFile",
"Ljava/lang/String;"));
- mvpfile.filename = e->GetStringUTFChars(mvp, 0);
+ mvpfile.filename = strdup(e->GetStringUTFChars(mvp, 0));
int i;
for(i = 0; i < sizeof(hashes)/sizeof(hashes[0]); i++)
{
@@ -274,7 +276,7 @@ JNIEXPORT jobjectArray JNICALL Java_MVPTree_query
}
}
- DP *query = ph_malloc_datapoint(mvpfile.hash_type, mvpfile.pathlength);
+ DP *query = ph_malloc_datapoint(mvpfile.hash_type);
DP **results = (DP **)malloc(max*sizeof(DP *));
const char *hash_file = NULL;
jstring hashStr = (jstring)e->GetObjectField(hashObj, hash_filename);
@@ -321,7 +323,8 @@ JNIEXPORT jobjectArray JNICALL Java_MVPTree_query
break;
}
}
- int res = ph_query_mvptree(&mvpfile, query, max, radius, results, &count);
+ int res = ph_query_mvptree(&mvpfile, query, max, radius,
+ thresh, results, count);
if(type == AUDIO_HASH)
e->ReleaseIntArrayElements(hashList, hash_list, JNI_ABORT);
jobjectArray ret;
@@ -374,6 +377,7 @@ JNIEXPORT jobjectArray JNICALL Java_MVPTree_query
ph_free_datapoint(results[i]);
}
free(results);
+ free(mvpfile.filename);
return ret;
}
@@ -391,7 +395,7 @@ JNIEXPORT jboolean JNICALL Java_MVPTree_add
jstring mvp = (jstring)e->GetObjectField(ob, e->GetFieldID(e->FindClass("MVPTree"), "mvpFile",
"Ljava/lang/String;"));
- mvpfile.filename = e->GetStringUTFChars(mvp, 0);
+ mvpfile.filename = strdup(e->GetStringUTFChars(mvp, 0));
int i;
jobject hashObj = e->GetObjectArrayElement(hashArray, 0);
for(i = 0; i < sizeof(hashes)/sizeof(hashes[0]); i++)
@@ -413,7 +417,7 @@ JNIEXPORT jboolean JNICALL Java_MVPTree_add
for(int j = 0; j < len; j++)
{
- newHashes[j] = ph_malloc_datapoint(mvpfile.hash_type, mvpfile.pathlength);
+ newHashes[j] = ph_malloc_datapoint(mvpfile.hash_type);
hashObj = e->GetObjectArrayElement(hashArray, j);
jstring hashStr = (jstring)e->GetObjectField(hashObj, hash_filename);
@@ -465,7 +469,8 @@ JNIEXPORT jboolean JNICALL Java_MVPTree_add
}
- int res = ph_add_mvptree(&mvpfile, newHashes, len);
+ int nbsaved = 0;
+ int res = ph_add_mvptree(&mvpfile, newHashes, len, nbsaved);
for(int j = 0; j < len; j++)
{
@@ -480,6 +485,7 @@ JNIEXPORT jboolean JNICALL Java_MVPTree_add
e->ReleaseStringUTFChars(mvp, mvpfile.filename);
free(newHashes);
+ free(mvpfile.filename);
return JNI_TRUE;
}
@@ -692,7 +698,7 @@ JNIEXPORT jobject JNICALL Java_pHash_audioHash
float *buf = NULL;
unsigned int *hash = NULL;
const char *file = e->GetStringUTFChars(f,0);
- buf = ph_readaudio(file,sr,channels,N);
+ buf = ph_readaudio(file,sr,channels,NULL,N);
if(!buf)
{
e->ReleaseStringUTFChars(f,file);
diff --git a/bindings/java/pHash.java b/bindings/java/pHash.java
index 3bc03ae..daddf00 100644
--- a/bindings/java/pHash.java
+++ b/bindings/java/pHash.java
@@ -50,7 +50,7 @@ public class pHash
if(result)
{
System.out.println("Successfully created MVP tree");
- Hash[] results = mvp.query(hashes[0], 100, 20);
+ Hash[] results = mvp.query(hashes[0], 100, 20, 30);
if(results != null && results.length > 0)
{
System.out.println("Query found " + results.length + " results");
@@ -64,13 +64,14 @@ public class pHash
if(added)
{
System.out.println("Hashes added successfully.");
- Hash[] foundHashes = mvp.query(newHashes[0], 100, 20);
+ Hash[] foundHashes = mvp.query(newHashes[0], 100, 20, 30);
if(foundHashes != null && foundHashes.length > 0)
{
System.out.println("Found newly added hash.");
}
}
- }
+ } else
+ System.out.println("Creating tree failed");
}
else if(args[i].equals("-a"))
diff --git a/bindings/java/pHash_MVPTree.h b/bindings/java/pHash_MVPTree.h
index e96bfef..0b77e6c 100644
--- a/bindings/java/pHash_MVPTree.h
+++ b/bindings/java/pHash_MVPTree.h
@@ -21,7 +21,7 @@ JNIEXPORT jboolean JNICALL Java_MVPTree_create
* Signature: (LHash;FI)[LHash;
*/
JNIEXPORT jobjectArray JNICALL Java_MVPTree_query
- (JNIEnv *, jobject, jobject, jfloat, jint);
+ (JNIEnv *, jobject, jobject, jfloat, jfloat, jint);
/*
* Class: MVPTree
diff --git a/config.h.in b/config.h.in
index b4abd32..45621b8 100644
--- a/config.h.in
+++ b/config.h.in
@@ -1,5 +1,8 @@
/* config.h.in. Generated from configure.ac by autoheader. */
+/* configure with audio hash */
+#undef HAVE_AUDIO_HASH
+
/* Define to 1 if you have the <dlfcn.h> header file. */
#undef HAVE_DLFCN_H
@@ -9,6 +12,9 @@
/* Define to 1 if you have the `gettimeofday' function. */
#undef HAVE_GETTIMEOFDAY
+/* configure with image hash */
+#undef HAVE_IMAGE_HASH
+
/* Define to 1 if you have the <inttypes.h> header file. */
#undef HAVE_INTTYPES_H
@@ -24,9 +30,6 @@
/* Define to 1 if you have the `m' library (-lm). */
#undef HAVE_LIBM
-/* Define to 1 if you have the `pthread' library (-lpthread). */
-#undef HAVE_LIBPTHREAD
-
/* Define to 1 if you have the `swscale' library (-lswscale). */
#undef HAVE_LIBSWSCALE
@@ -51,6 +54,9 @@
/* Define to 1 if you have the `pow' function. */
#undef HAVE_POW
+/* configure with pthread support */
+#undef HAVE_PTHREAD
+
/* Define to 1 if you have the `realloc' function. */
#undef HAVE_REALLOC
@@ -93,6 +99,9 @@
/* Define to 1 if you have the <unistd.h> header file. */
#undef HAVE_UNISTD_H
+/* configure with video hash */
+#undef HAVE_VIDEO_HASH
+
/* Define to 1 if the system has the type `_Bool'. */
#undef HAVE__BOOL
@@ -118,6 +127,10 @@
/* Define to the version of this package. */
#undef PACKAGE_VERSION
+/* Define to necessary symbol if this constant uses a non-standard name on
+ your system. */
+#undef PTHREAD_CREATE_JOINABLE
+
/* Define to 1 if you have the ANSI C header files. */
#undef STDC_HEADERS
diff --git a/configure b/configure
index 4c5684f..fa6c599 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.63 for pHash 0.8.1.
+# Generated by GNU Autoconf 2.63 for pHash 0.9.0.
#
# Report bugs to <support at phash.org>.
#
@@ -745,8 +745,8 @@ SHELL=${CONFIG_SHELL-/bin/sh}
# Identity of this package.
PACKAGE_NAME='pHash'
PACKAGE_TARNAME='phash'
-PACKAGE_VERSION='0.8.1'
-PACKAGE_STRING='pHash 0.8.1'
+PACKAGE_VERSION='0.9.0'
+PACKAGE_STRING='pHash 0.9.0'
PACKAGE_BUGREPORT='support at phash.org'
ac_unique_file="src/pHash.cpp"
@@ -813,6 +813,19 @@ DUMPBIN
LD
FGREP
SED
+LIBTOOL
+LN_S
+WITH_JAVA_FALSE
+WITH_JAVA_TRUE
+EGREP
+GREP
+CPP
+HAVE_PTHREAD_FALSE
+HAVE_PTHREAD_TRUE
+PTHREAD_CFLAGS
+PTHREAD_LIBS
+PTHREAD_CC
+ax_pthread_config
host_os
host_vendor
host_cpu
@@ -821,13 +834,6 @@ build_os
build_vendor
build_cpu
build
-LIBTOOL
-LN_S
-WITH_JAVA_FALSE
-WITH_JAVA_TRUE
-EGREP
-GREP
-CPP
am__fastdepGCJ_FALSE
am__fastdepGCJ_TRUE
GCJDEPMODE
@@ -920,6 +926,7 @@ ac_subst_files=''
ac_user_opts='
enable_option_checking
enable_dependency_tracking
+enable_pthread
enable_debug
enable_openmp
enable_java
@@ -1498,7 +1505,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures pHash 0.8.1 to adapt to many kinds of systems.
+\`configure' configures pHash 0.9.0 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1568,7 +1575,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of pHash 0.8.1:";;
+ short | recursive ) echo "Configuration of pHash 0.9.0:";;
esac
cat <<\_ACEOF
@@ -1578,6 +1585,7 @@ Optional Features:
--enable-FEATURE[=ARG] include FEATURE [ARG=yes]
--disable-dependency-tracking speeds up one-time build
--enable-dependency-tracking do not reject slow dependency extractors
+ --disable-pthread pthread support [default=no]
--enable-debug compile with debugging support [default=no]
--enable-openmp enable OpenMP support in pHash to use multiple
cores/CPUs [default=no]
@@ -1680,7 +1688,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-pHash configure 0.8.1
+pHash configure 0.9.0
generated by GNU Autoconf 2.63
Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
@@ -1694,7 +1702,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by pHash $as_me 0.8.1, which was
+It was created by pHash $as_me 0.9.0, which was
generated by GNU Autoconf 2.63. Invocation command line was
$ $0 $@
@@ -2588,6 +2596,106 @@ ac_config_headers="$ac_config_headers config.h"
# Checks for programs.
+# ===========================================================================
+# http://www.nongnu.org/autoconf-archive/ax_pthread.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]])
+#
+# DESCRIPTION
+#
+# This macro figures out how to build C programs using POSIX threads. It
+# sets the PTHREAD_LIBS output variable to the threads library and linker
+# flags, and the PTHREAD_CFLAGS output variable to any special C compiler
+# flags that are needed. (The user can also force certain compiler
+# flags/libs to be tested by setting these environment variables.)
+#
+# Also sets PTHREAD_CC to any special C compiler that is needed for
+# multi-threaded programs (defaults to the value of CC otherwise). (This
+# is necessary on AIX to use the special cc_r compiler alias.)
+#
+# NOTE: You are assumed to not only compile your program with these flags,
+# but also link it with them as well. e.g. you should link with
+# $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS
+#
+# If you are only building threads programs, you may wish to use these
+# variables in your default LIBS, CFLAGS, and CC:
+#
+# LIBS="$PTHREAD_LIBS $LIBS"
+# CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+# CC="$PTHREAD_CC"
+#
+# In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute constant
+# has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to that name
+# (e.g. PTHREAD_CREATE_UNDETACHED on AIX).
+#
+# ACTION-IF-FOUND is a list of shell commands to run if a threads library
+# is found, and ACTION-IF-NOT-FOUND is a list of commands to run it if it
+# is not found. If ACTION-IF-FOUND is not specified, the default action
+# will define HAVE_PTHREAD.
+#
+# Please let the authors know if this macro fails on any platform, or if
+# you have any other suggestions or comments. This macro was based on work
+# by SGJ on autoconf scripts for FFTW (http://www.fftw.org/) (with help
+# from M. Frigo), as well as ac_pthread and hb_pthread macros posted by
+# Alejandro Forero Cuervo to the autoconf macro repository. We are also
+# grateful for the helpful feedback of numerous users.
+#
+# LICENSE
+#
+# Copyright (c) 2008 Steven G. Johnson <stevenj at alum.mit.edu>
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# As a special exception, the respective Autoconf Macro's copyright owner
+# gives unlimited permission to copy, distribute and modify the configure
+# scripts that are the output of Autoconf when processing the Macro. You
+# need not follow the terms of the GNU General Public License when using
+# or distributing such scripts, even though portions of the text of the
+# Macro appear in them. The GNU General Public License (GPL) does govern
+# all other use of the material that constitutes the Autoconf Macro.
+#
+# This special exception to the GPL applies to versions of the Autoconf
+# Macro released by the Autoconf Archive. When you make and distribute a
+# modified version of the Autoconf Macro, you may extend this special
+# exception to the GPL to apply to your modified version as well.
+
+#serial 6
+
+# This is what autoupdate's m4 run will expand. It fires
+# the warning (with _au_warn_XXX), outputs it into the
+# updated configure.ac (with AC_DIAGNOSE), and then outputs
+# the replacement expansion.
+
+
+# This is an auxiliary macro that is also run when
+# autoupdate runs m4. It simply calls m4_warning, but
+# we need a wrapper so that each warning is emitted only
+# once. We break the quoting in m4_warning's argument in
+# order to expand this macro's arguments, not AU_DEFUN's.
+
+
+# Finally, this is the expansion that is picked up by
+# autoconf. It tells the user to run autoupdate, and
+# then outputs the replacement expansion. We do not care
+# about autoupdate's warning because that contains
+# information on what to do *after* running autoupdate.
+
+
+
ac_ext=cpp
ac_cpp='$CXXCPP $CPPFLAGS'
ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
@@ -4445,6 +4553,674 @@ fi
+if test -z $DISTRO; then
+ { $as_echo "$as_me:$LINENO: checking for /etc/gentoo-release" >&5
+$as_echo_n "checking for /etc/gentoo-release... " >&6; }
+if test "${ac_cv_file__etc_gentoo_release+set}" = set; then
+ $as_echo_n "(cached) " >&6
+else
+ test "$cross_compiling" = yes &&
+ { { $as_echo "$as_me:$LINENO: error: cannot check for file existence when cross compiling" >&5
+$as_echo "$as_me: error: cannot check for file existence when cross compiling" >&2;}
+ { (exit 1); exit 1; }; }
+if test -r "/etc/gentoo-release"; then
+ ac_cv_file__etc_gentoo_release=yes
+else
+ ac_cv_file__etc_gentoo_release=no
+fi
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_file__etc_gentoo_release" >&5
+$as_echo "$ac_cv_file__etc_gentoo_release" >&6; }
+if test "x$ac_cv_file__etc_gentoo_release" = x""yes; then
+ DISTRO="gentoo"
+fi
+
+ { $as_echo "$as_me:$LINENO: checking for /etc/redhat-release" >&5
+$as_echo_n "checking for /etc/redhat-release... " >&6; }
+if test "${ac_cv_file__etc_redhat_release+set}" = set; then
+ $as_echo_n "(cached) " >&6
+else
+ test "$cross_compiling" = yes &&
+ { { $as_echo "$as_me:$LINENO: error: cannot check for file existence when cross compiling" >&5
+$as_echo "$as_me: error: cannot check for file existence when cross compiling" >&2;}
+ { (exit 1); exit 1; }; }
+if test -r "/etc/redhat-release"; then
+ ac_cv_file__etc_redhat_release=yes
+else
+ ac_cv_file__etc_redhat_release=no
+fi
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_file__etc_redhat_release" >&5
+$as_echo "$ac_cv_file__etc_redhat_release" >&6; }
+if test "x$ac_cv_file__etc_redhat_release" = x""yes; then
+ DISTRO="redhat"
+fi
+
+ { $as_echo "$as_me:$LINENO: checking for /etc/slackware-version" >&5
+$as_echo_n "checking for /etc/slackware-version... " >&6; }
+if test "${ac_cv_file__etc_slackware_version+set}" = set; then
+ $as_echo_n "(cached) " >&6
+else
+ test "$cross_compiling" = yes &&
+ { { $as_echo "$as_me:$LINENO: error: cannot check for file existence when cross compiling" >&5
+$as_echo "$as_me: error: cannot check for file existence when cross compiling" >&2;}
+ { (exit 1); exit 1; }; }
+if test -r "/etc/slackware-version"; then
+ ac_cv_file__etc_slackware_version=yes
+else
+ ac_cv_file__etc_slackware_version=no
+fi
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_file__etc_slackware_version" >&5
+$as_echo "$ac_cv_file__etc_slackware_version" >&6; }
+if test "x$ac_cv_file__etc_slackware_version" = x""yes; then
+ DISTRO="slackware"
+fi
+
+ { $as_echo "$as_me:$LINENO: checking for /etc/debian_version" >&5
+$as_echo_n "checking for /etc/debian_version... " >&6; }
+if test "${ac_cv_file__etc_debian_version+set}" = set; then
+ $as_echo_n "(cached) " >&6
+else
+ test "$cross_compiling" = yes &&
+ { { $as_echo "$as_me:$LINENO: error: cannot check for file existence when cross compiling" >&5
+$as_echo "$as_me: error: cannot check for file existence when cross compiling" >&2;}
+ { (exit 1); exit 1; }; }
+if test -r "/etc/debian_version"; then
+ ac_cv_file__etc_debian_version=yes
+else
+ ac_cv_file__etc_debian_version=no
+fi
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_file__etc_debian_version" >&5
+$as_echo "$ac_cv_file__etc_debian_version" >&6; }
+if test "x$ac_cv_file__etc_debian_version" = x""yes; then
+ DISTRO="debian"
+fi
+
+ { $as_echo "$as_me:$LINENO: checking for /etc/SuSErelease" >&5
+$as_echo_n "checking for /etc/SuSErelease... " >&6; }
+if test "${ac_cv_file__etc_SuSErelease+set}" = set; then
+ $as_echo_n "(cached) " >&6
+else
+ test "$cross_compiling" = yes &&
+ { { $as_echo "$as_me:$LINENO: error: cannot check for file existence when cross compiling" >&5
+$as_echo "$as_me: error: cannot check for file existence when cross compiling" >&2;}
+ { (exit 1); exit 1; }; }
+if test -r "/etc/SuSErelease"; then
+ ac_cv_file__etc_SuSErelease=yes
+else
+ ac_cv_file__etc_SuSErelease=no
+fi
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_file__etc_SuSErelease" >&5
+$as_echo "$ac_cv_file__etc_SuSErelease" >&6; }
+if test "x$ac_cv_file__etc_SuSErelease" = x""yes; then
+ DISTRO="suse"
+fi
+
+fi
+
+
+# Check whether --enable-pthread was given.
+if test "${enable_pthread+set}" = set; then
+ enableval=$enable_pthread; PTHREAD="$enableval"
+else
+ PTHREAD="yes"
+fi
+
+
+if test "$PTHREAD" = "yes"; then
+ # Make sure we can run config.sub.
+$SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 ||
+ { { $as_echo "$as_me:$LINENO: error: cannot run $SHELL $ac_aux_dir/config.sub" >&5
+$as_echo "$as_me: error: cannot run $SHELL $ac_aux_dir/config.sub" >&2;}
+ { (exit 1); exit 1; }; }
+
+{ $as_echo "$as_me:$LINENO: checking build system type" >&5
+$as_echo_n "checking build system type... " >&6; }
+if test "${ac_cv_build+set}" = set; then
+ $as_echo_n "(cached) " >&6
+else
+ ac_build_alias=$build_alias
+test "x$ac_build_alias" = x &&
+ ac_build_alias=`$SHELL "$ac_aux_dir/config.guess"`
+test "x$ac_build_alias" = x &&
+ { { $as_echo "$as_me:$LINENO: error: cannot guess build type; you must specify one" >&5
+$as_echo "$as_me: error: cannot guess build type; you must specify one" >&2;}
+ { (exit 1); exit 1; }; }
+ac_cv_build=`$SHELL "$ac_aux_dir/config.sub" $ac_build_alias` ||
+ { { $as_echo "$as_me:$LINENO: error: $SHELL $ac_aux_dir/config.sub $ac_build_alias failed" >&5
+$as_echo "$as_me: error: $SHELL $ac_aux_dir/config.sub $ac_build_alias failed" >&2;}
+ { (exit 1); exit 1; }; }
+
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_build" >&5
+$as_echo "$ac_cv_build" >&6; }
+case $ac_cv_build in
+*-*-*) ;;
+*) { { $as_echo "$as_me:$LINENO: error: invalid value of canonical build" >&5
+$as_echo "$as_me: error: invalid value of canonical build" >&2;}
+ { (exit 1); exit 1; }; };;
+esac
+build=$ac_cv_build
+ac_save_IFS=$IFS; IFS='-'
+set x $ac_cv_build
+shift
+build_cpu=$1
+build_vendor=$2
+shift; shift
+# Remember, the first character of IFS is used to create $*,
+# except with old shells:
+build_os=$*
+IFS=$ac_save_IFS
+case $build_os in *\ *) build_os=`echo "$build_os" | sed 's/ /-/g'`;; esac
+
+
+{ $as_echo "$as_me:$LINENO: checking host system type" >&5
+$as_echo_n "checking host system type... " >&6; }
+if test "${ac_cv_host+set}" = set; then
+ $as_echo_n "(cached) " >&6
+else
+ if test "x$host_alias" = x; then
+ ac_cv_host=$ac_cv_build
+else
+ ac_cv_host=`$SHELL "$ac_aux_dir/config.sub" $host_alias` ||
+ { { $as_echo "$as_me:$LINENO: error: $SHELL $ac_aux_dir/config.sub $host_alias failed" >&5
+$as_echo "$as_me: error: $SHELL $ac_aux_dir/config.sub $host_alias failed" >&2;}
+ { (exit 1); exit 1; }; }
+fi
+
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_host" >&5
+$as_echo "$ac_cv_host" >&6; }
+case $ac_cv_host in
+*-*-*) ;;
+*) { { $as_echo "$as_me:$LINENO: error: invalid value of canonical host" >&5
+$as_echo "$as_me: error: invalid value of canonical host" >&2;}
+ { (exit 1); exit 1; }; };;
+esac
+host=$ac_cv_host
+ac_save_IFS=$IFS; IFS='-'
+set x $ac_cv_host
+shift
+host_cpu=$1
+host_vendor=$2
+shift; shift
+# Remember, the first character of IFS is used to create $*,
+# except with old shells:
+host_os=$*
+IFS=$ac_save_IFS
+case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac
+
+
+
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+ax_pthread_ok=no
+
+# We used to check for pthread.h first, but this fails if pthread.h
+# requires special compiler flags (e.g. on True64 or Sequent).
+# It gets checked for in the link test anyway.
+
+# First of all, check if the user has set any of the PTHREAD_LIBS,
+# etcetera environment variables, and if threads linking works using
+# them:
+if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then
+ save_CFLAGS="$CFLAGS"
+ CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+ save_LIBS="$LIBS"
+ LIBS="$PTHREAD_LIBS $LIBS"
+ { $as_echo "$as_me:$LINENO: checking for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS" >&5
+$as_echo_n "checking for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS... " >&6; }
+ cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h. */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h. */
+
+/* Override any GCC internal prototype to avoid an error.
+ Use char because int might match the return type of a GCC
+ builtin and then its argument prototype would still apply. */
+#ifdef __cplusplus
+extern "C"
+#endif
+char pthread_join ();
+int
+main ()
+{
+return pthread_join ();
+ ;
+ return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (ac_try="$ac_link"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+ (eval "$ac_link") 2>conftest.er1
+ ac_status=$?
+ grep -v '^ *+' conftest.er1 >conftest.err
+ rm -f conftest.er1
+ cat conftest.err >&5
+ $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); } && {
+ test -z "$ac_c_werror_flag" ||
+ test ! -s conftest.err
+ } && test -s conftest$ac_exeext && {
+ test "$cross_compiling" = yes ||
+ $as_test_x conftest$ac_exeext
+ }; then
+ ax_pthread_ok=yes
+else
+ $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+
+fi
+
+rm -rf conftest.dSYM
+rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \
+ conftest$ac_exeext conftest.$ac_ext
+ { $as_echo "$as_me:$LINENO: result: $ax_pthread_ok" >&5
+$as_echo "$ax_pthread_ok" >&6; }
+ if test x"$ax_pthread_ok" = xno; then
+ PTHREAD_LIBS=""
+ PTHREAD_CFLAGS=""
+ fi
+ LIBS="$save_LIBS"
+ CFLAGS="$save_CFLAGS"
+fi
+
+# We must check for the threads library under a number of different
+# names; the ordering is very important because some systems
+# (e.g. DEC) have both -lpthread and -lpthreads, where one of the
+# libraries is broken (non-POSIX).
+
+# Create a list of thread flags to try. Items starting with a "-" are
+# C compiler flags, and other items are library names, except for "none"
+# which indicates that we try without any flags at all, and "pthread-config"
+# which is a program returning the flags for the Pth emulation library.
+
+ax_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config"
+
+# The ordering *is* (sometimes) important. Some notes on the
+# individual items follow:
+
+# pthreads: AIX (must check this before -lpthread)
+# none: in case threads are in libc; should be tried before -Kthread and
+# other compiler flags to prevent continual compiler warnings
+# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h)
+# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able)
+# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread)
+# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads)
+# -pthreads: Solaris/gcc
+# -mthreads: Mingw32/gcc, Lynx/gcc
+# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it
+# doesn't hurt to check since this sometimes defines pthreads too;
+# also defines -D_REENTRANT)
+# ... -mt is also the pthreads flag for HP/aCC
+# pthread: Linux, etcetera
+# --thread-safe: KAI C++
+# pthread-config: use pthread-config program (for GNU Pth library)
+
+case "${host_cpu}-${host_os}" in
+ *solaris*)
+
+ # On Solaris (at least, for some versions), libc contains stubbed
+ # (non-functional) versions of the pthreads routines, so link-based
+ # tests will erroneously succeed. (We need to link with -pthreads/-mt/
+ # -lpthread.) (The stubs are missing pthread_cleanup_push, or rather
+ # a function called by this macro, so we could check for that, but
+ # who knows whether they'll stub that too in a future libc.) So,
+ # we'll just look for -pthreads and -lpthread first:
+
+ ax_pthread_flags="-pthreads pthread -mt -pthread $ax_pthread_flags"
+ ;;
+
+ *-darwin*)
+ acx_pthread_flags="-pthread $acx_pthread_flags"
+ ;;
+esac
+
+if test x"$ax_pthread_ok" = xno; then
+for flag in $ax_pthread_flags; do
+
+ case $flag in
+ none)
+ { $as_echo "$as_me:$LINENO: checking whether pthreads work without any flags" >&5
+$as_echo_n "checking whether pthreads work without any flags... " >&6; }
+ ;;
+
+ -*)
+ { $as_echo "$as_me:$LINENO: checking whether pthreads work with $flag" >&5
+$as_echo_n "checking whether pthreads work with $flag... " >&6; }
+ PTHREAD_CFLAGS="$flag"
+ ;;
+
+ pthread-config)
+ # Extract the first word of "pthread-config", so it can be a program name with args.
+set dummy pthread-config; ac_word=$2
+{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_ax_pthread_config+set}" = set; then
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$ax_pthread_config"; then
+ ac_cv_prog_ax_pthread_config="$ax_pthread_config" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_ax_pthread_config="yes"
+ $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+done
+IFS=$as_save_IFS
+
+ test -z "$ac_cv_prog_ax_pthread_config" && ac_cv_prog_ax_pthread_config="no"
+fi
+fi
+ax_pthread_config=$ac_cv_prog_ax_pthread_config
+if test -n "$ax_pthread_config"; then
+ { $as_echo "$as_me:$LINENO: result: $ax_pthread_config" >&5
+$as_echo "$ax_pthread_config" >&6; }
+else
+ { $as_echo "$as_me:$LINENO: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ if test x"$ax_pthread_config" = xno; then continue; fi
+ PTHREAD_CFLAGS="`pthread-config --cflags`"
+ PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`"
+ ;;
+
+ *)
+ { $as_echo "$as_me:$LINENO: checking for the pthreads library -l$flag" >&5
+$as_echo_n "checking for the pthreads library -l$flag... " >&6; }
+ PTHREAD_LIBS="-l$flag"
+ ;;
+ esac
+
+ save_LIBS="$LIBS"
+ save_CFLAGS="$CFLAGS"
+ LIBS="$PTHREAD_LIBS $LIBS"
+ CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+
+ # Check for various functions. We must include pthread.h,
+ # since some functions may be macros. (On the Sequent, we
+ # need a special flag -Kthread to make this header compile.)
+ # We check for pthread_join because it is in -lpthread on IRIX
+ # while pthread_create is in libc. We check for pthread_attr_init
+ # due to DEC craziness with -lpthreads. We check for
+ # pthread_cleanup_push because it is one of the few pthread
+ # functions on Solaris that doesn't have a non-functional libc stub.
+ # We try pthread_create on general principles.
+ cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h. */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h. */
+#include <pthread.h>
+ static void routine(void* a) {a=0;}
+ static void* start_routine(void* a) {return a;}
+int
+main ()
+{
+pthread_t th; pthread_attr_t attr;
+ pthread_join(th, 0);
+ pthread_attr_init(&attr);
+ pthread_cleanup_push(routine, 0);
+ pthread_create(&th,0,start_routine,0);
+ pthread_cleanup_pop(0);
+ ;
+ return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (ac_try="$ac_link"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+ (eval "$ac_link") 2>conftest.er1
+ ac_status=$?
+ grep -v '^ *+' conftest.er1 >conftest.err
+ rm -f conftest.er1
+ cat conftest.err >&5
+ $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); } && {
+ test -z "$ac_c_werror_flag" ||
+ test ! -s conftest.err
+ } && test -s conftest$ac_exeext && {
+ test "$cross_compiling" = yes ||
+ $as_test_x conftest$ac_exeext
+ }; then
+ ax_pthread_ok=yes
+else
+ $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+
+fi
+
+rm -rf conftest.dSYM
+rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \
+ conftest$ac_exeext conftest.$ac_ext
+
+ LIBS="$save_LIBS"
+ CFLAGS="$save_CFLAGS"
+
+ { $as_echo "$as_me:$LINENO: result: $ax_pthread_ok" >&5
+$as_echo "$ax_pthread_ok" >&6; }
+ if test "x$ax_pthread_ok" = xyes; then
+ break;
+ fi
+
+ PTHREAD_LIBS=""
+ PTHREAD_CFLAGS=""
+done
+fi
+
+# Various other checks:
+if test "x$ax_pthread_ok" = xyes; then
+ save_LIBS="$LIBS"
+ LIBS="$PTHREAD_LIBS $LIBS"
+ save_CFLAGS="$CFLAGS"
+ CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+
+ # Detect AIX lossage: JOINABLE attribute is called UNDETACHED.
+ { $as_echo "$as_me:$LINENO: checking for joinable pthread attribute" >&5
+$as_echo_n "checking for joinable pthread attribute... " >&6; }
+ attr_name=unknown
+ for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do
+ cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h. */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h. */
+#include <pthread.h>
+int
+main ()
+{
+int attr=$attr; return attr;
+ ;
+ return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext conftest$ac_exeext
+if { (ac_try="$ac_link"
+case "(($ac_try" in
+ *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+ *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+ (eval "$ac_link") 2>conftest.er1
+ ac_status=$?
+ grep -v '^ *+' conftest.er1 >conftest.err
+ rm -f conftest.er1
+ cat conftest.err >&5
+ $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+ (exit $ac_status); } && {
+ test -z "$ac_c_werror_flag" ||
+ test ! -s conftest.err
+ } && test -s conftest$ac_exeext && {
+ test "$cross_compiling" = yes ||
+ $as_test_x conftest$ac_exeext
+ }; then
+ attr_name=$attr; break
+else
+ $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+
+fi
+
+rm -rf conftest.dSYM
+rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \
+ conftest$ac_exeext conftest.$ac_ext
+ done
+ { $as_echo "$as_me:$LINENO: result: $attr_name" >&5
+$as_echo "$attr_name" >&6; }
+ if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then
+
+cat >>confdefs.h <<_ACEOF
+#define PTHREAD_CREATE_JOINABLE $attr_name
+_ACEOF
+
+ fi
+
+ { $as_echo "$as_me:$LINENO: checking if more special flags are required for pthreads" >&5
+$as_echo_n "checking if more special flags are required for pthreads... " >&6; }
+ flag=no
+ case "${host_cpu}-${host_os}" in
+ *-aix* | *-freebsd* | *-darwin*) flag="-D_THREAD_SAFE";;
+ *solaris* | *-osf* | *-hpux*) flag="-D_REENTRANT";;
+ esac
+ { $as_echo "$as_me:$LINENO: result: ${flag}" >&5
+$as_echo "${flag}" >&6; }
+ if test "x$flag" != xno; then
+ PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS"
+ fi
+
+ LIBS="$save_LIBS"
+ CFLAGS="$save_CFLAGS"
+
+ # More AIX lossage: must compile with xlc_r or cc_r
+ if test x"$GCC" != xyes; then
+ for ac_prog in xlc_r cc_r
+do
+ # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ $as_echo "$as_me:$LINENO: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if test "${ac_cv_prog_PTHREAD_CC+set}" = set; then
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$PTHREAD_CC"; then
+ ac_cv_prog_PTHREAD_CC="$PTHREAD_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+ ac_cv_prog_PTHREAD_CC="$ac_prog"
+ $as_echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+done
+IFS=$as_save_IFS
+
+fi
+fi
+PTHREAD_CC=$ac_cv_prog_PTHREAD_CC
+if test -n "$PTHREAD_CC"; then
+ { $as_echo "$as_me:$LINENO: result: $PTHREAD_CC" >&5
+$as_echo "$PTHREAD_CC" >&6; }
+else
+ { $as_echo "$as_me:$LINENO: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+ test -n "$PTHREAD_CC" && break
+done
+test -n "$PTHREAD_CC" || PTHREAD_CC="${CC}"
+
+ else
+ PTHREAD_CC=$CC
+ fi
+else
+ PTHREAD_CC="$CC"
+fi
+
+
+
+
+
+# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND:
+if test x"$ax_pthread_ok" = xyes; then
+
+ LIBS="$PTHREAD_LIBS $LIBS"
+ CPPFLAGS="$CLFAGS $PTHREAD_CFLAGS"
+ CC="$PTHREAD_CC"
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_PTHREAD 1
+_ACEOF
+
+
+ :
+else
+ ax_pthread_ok=no
+
+fi
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+fi
+
+ if test "$PTHREAD" = "yes"; then
+ HAVE_PTHREAD_TRUE=
+ HAVE_PTHREAD_FALSE='#'
+else
+ HAVE_PTHREAD_TRUE='#'
+ HAVE_PTHREAD_FALSE=
+fi
+
+
# Check whether --enable-debug was given.
if test "${enable_debug+set}" = set; then
enableval=$enable_debug; debug=$enableval
@@ -4487,7 +5263,6 @@ else
fi
-
ac_ext=c
ac_cpp='$CPP $CPPFLAGS'
ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
@@ -5429,89 +6204,6 @@ macro_revision='1.3017'
ltmain="$ac_aux_dir/ltmain.sh"
-# Make sure we can run config.sub.
-$SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 ||
- { { $as_echo "$as_me:$LINENO: error: cannot run $SHELL $ac_aux_dir/config.sub" >&5
-$as_echo "$as_me: error: cannot run $SHELL $ac_aux_dir/config.sub" >&2;}
- { (exit 1); exit 1; }; }
-
-{ $as_echo "$as_me:$LINENO: checking build system type" >&5
-$as_echo_n "checking build system type... " >&6; }
-if test "${ac_cv_build+set}" = set; then
- $as_echo_n "(cached) " >&6
-else
- ac_build_alias=$build_alias
-test "x$ac_build_alias" = x &&
- ac_build_alias=`$SHELL "$ac_aux_dir/config.guess"`
-test "x$ac_build_alias" = x &&
- { { $as_echo "$as_me:$LINENO: error: cannot guess build type; you must specify one" >&5
-$as_echo "$as_me: error: cannot guess build type; you must specify one" >&2;}
- { (exit 1); exit 1; }; }
-ac_cv_build=`$SHELL "$ac_aux_dir/config.sub" $ac_build_alias` ||
- { { $as_echo "$as_me:$LINENO: error: $SHELL $ac_aux_dir/config.sub $ac_build_alias failed" >&5
-$as_echo "$as_me: error: $SHELL $ac_aux_dir/config.sub $ac_build_alias failed" >&2;}
- { (exit 1); exit 1; }; }
-
-fi
-{ $as_echo "$as_me:$LINENO: result: $ac_cv_build" >&5
-$as_echo "$ac_cv_build" >&6; }
-case $ac_cv_build in
-*-*-*) ;;
-*) { { $as_echo "$as_me:$LINENO: error: invalid value of canonical build" >&5
-$as_echo "$as_me: error: invalid value of canonical build" >&2;}
- { (exit 1); exit 1; }; };;
-esac
-build=$ac_cv_build
-ac_save_IFS=$IFS; IFS='-'
-set x $ac_cv_build
-shift
-build_cpu=$1
-build_vendor=$2
-shift; shift
-# Remember, the first character of IFS is used to create $*,
-# except with old shells:
-build_os=$*
-IFS=$ac_save_IFS
-case $build_os in *\ *) build_os=`echo "$build_os" | sed 's/ /-/g'`;; esac
-
-
-{ $as_echo "$as_me:$LINENO: checking host system type" >&5
-$as_echo_n "checking host system type... " >&6; }
-if test "${ac_cv_host+set}" = set; then
- $as_echo_n "(cached) " >&6
-else
- if test "x$host_alias" = x; then
- ac_cv_host=$ac_cv_build
-else
- ac_cv_host=`$SHELL "$ac_aux_dir/config.sub" $host_alias` ||
- { { $as_echo "$as_me:$LINENO: error: $SHELL $ac_aux_dir/config.sub $host_alias failed" >&5
-$as_echo "$as_me: error: $SHELL $ac_aux_dir/config.sub $host_alias failed" >&2;}
- { (exit 1); exit 1; }; }
-fi
-
-fi
-{ $as_echo "$as_me:$LINENO: result: $ac_cv_host" >&5
-$as_echo "$ac_cv_host" >&6; }
-case $ac_cv_host in
-*-*-*) ;;
-*) { { $as_echo "$as_me:$LINENO: error: invalid value of canonical host" >&5
-$as_echo "$as_me: error: invalid value of canonical host" >&2;}
- { (exit 1); exit 1; }; };;
-esac
-host=$ac_cv_host
-ac_save_IFS=$IFS; IFS='-'
-set x $ac_cv_host
-shift
-host_cpu=$1
-host_vendor=$2
-shift; shift
-# Remember, the first character of IFS is used to create $*,
-# except with old shells:
-host_os=$*
-IFS=$ac_save_IFS
-case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac
-
-
{ $as_echo "$as_me:$LINENO: checking for a sed that does not truncate output" >&5
$as_echo_n "checking for a sed that does not truncate output... " >&6; }
if test "${ac_cv_path_SED+set}" = set; then
@@ -5972,13 +6664,13 @@ if test "${lt_cv_nm_interface+set}" = set; then
else
lt_cv_nm_interface="BSD nm"
echo "int some_variable = 0;" > conftest.$ac_ext
- (eval echo "\"\$as_me:5975: $ac_compile\"" >&5)
+ (eval echo "\"\$as_me:6667: $ac_compile\"" >&5)
(eval "$ac_compile" 2>conftest.err)
cat conftest.err >&5
- (eval echo "\"\$as_me:5978: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+ (eval echo "\"\$as_me:6670: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
(eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
cat conftest.err >&5
- (eval echo "\"\$as_me:5981: output\"" >&5)
+ (eval echo "\"\$as_me:6673: output\"" >&5)
cat conftest.out >&5
if $GREP 'External.*some_variable' conftest.out > /dev/null; then
lt_cv_nm_interface="MS dumpbin"
@@ -7172,7 +7864,7 @@ ia64-*-hpux*)
;;
*-*-irix6*)
# Find out which ABI we are using.
- echo '#line 7175 "configure"' > conftest.$ac_ext
+ echo '#line 7867 "configure"' > conftest.$ac_ext
if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
(eval $ac_compile) 2>&5
ac_status=$?
@@ -9510,11 +10202,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:9513: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:10205: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:9517: \$? = $ac_status" >&5
+ echo "$as_me:10209: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -9849,11 +10541,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:9852: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:10544: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:9856: \$? = $ac_status" >&5
+ echo "$as_me:10548: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -9954,11 +10646,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:9957: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:10649: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:9961: \$? = $ac_status" >&5
+ echo "$as_me:10653: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -10009,11 +10701,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:10012: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:10704: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:10016: \$? = $ac_status" >&5
+ echo "$as_me:10708: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -12812,7 +13504,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 12815 "configure"
+#line 13507 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -12908,7 +13600,7 @@ else
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 12911 "configure"
+#line 13603 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -14928,11 +15620,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:14931: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:15623: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:14935: \$? = $ac_status" >&5
+ echo "$as_me:15627: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -15027,11 +15719,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:15030: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:15722: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:15034: \$? = $ac_status" >&5
+ echo "$as_me:15726: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -15079,11 +15771,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:15082: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:15774: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:15086: \$? = $ac_status" >&5
+ echo "$as_me:15778: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -16147,11 +16839,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:16150: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:16842: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:16154: \$? = $ac_status" >&5
+ echo "$as_me:16846: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -16480,11 +17172,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:16483: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:17175: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:16487: \$? = $ac_status" >&5
+ echo "$as_me:17179: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -16579,11 +17271,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:16582: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:17274: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:16586: \$? = $ac_status" >&5
+ echo "$as_me:17278: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -16631,11 +17323,11 @@ else
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:16634: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:17326: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:16638: \$? = $ac_status" >&5
+ echo "$as_me:17330: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -18144,9 +18836,15 @@ $as_echo "no" >&6; }
fi
fi
- CPPFLAGS="$CPPFLAGS -DHAVE_IMAGE_HASH=1"
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_IMAGE_HASH 1
+_ACEOF
+
+
fi
+
if test x$image_hash != xno; then
HAVE_IMAGE_HASH_TRUE=
HAVE_IMAGE_HASH_FALSE='#'
@@ -18674,9 +19372,15 @@ $as_echo "no" >&6; }
fi
fi
- CPPFLAGS="$CPPFLAGS -DHAVE_VIDEO_HASH=1"
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_VIDEO_HASH 1
+_ACEOF
+
+
fi
+
if test x$video_hash != xno; then
HAVE_VIDEO_HASH_TRUE=
HAVE_VIDEO_HASH_FALSE='#'
@@ -19047,7 +19751,12 @@ fi
found_ffmpeg="y"
fi
- CPPFLAGS="$CPPFLAGS -DHAVE_AUDIO_HASH=1"
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_AUDIO_HASH 1
+_ACEOF
+
+
fi
@@ -19061,6 +19770,16 @@ else
fi
+if test "$audio_hash" = "yes" -o "$video_hash" = "yes"; then
+case "$DISTRO" in
+ debian)
+ CPPFLAGS="$CPPFLAGS -I/usr/include/ffmpeg"
+ ;;
+ redhat)
+ CPPFLAGS="$CPPFLAGS -I/usr/include/libavcodec -I/usr/include/libavformat -I/usr/include/libswscale"
+ ;;
+esac
+fi
# Checks for libraries.
@@ -19278,81 +19997,6 @@ if test "x$ac_cv_lib_png_png_create_read_struct" = x""yes; then
fi
-{ $as_echo "$as_me:$LINENO: checking for pthread_create in -lpthread" >&5
-$as_echo_n "checking for pthread_create in -lpthread... " >&6; }
-if test "${ac_cv_lib_pthread_pthread_create+set}" = set; then
- $as_echo_n "(cached) " >&6
-else
- ac_check_lib_save_LIBS=$LIBS
-LIBS="-lpthread $LIBS"
-cat >conftest.$ac_ext <<_ACEOF
-/* confdefs.h. */
-_ACEOF
-cat confdefs.h >>conftest.$ac_ext
-cat >>conftest.$ac_ext <<_ACEOF
-/* end confdefs.h. */
-
-/* Override any GCC internal prototype to avoid an error.
- Use char because int might match the return type of a GCC
- builtin and then its argument prototype would still apply. */
-#ifdef __cplusplus
-extern "C"
-#endif
-char pthread_create ();
-int
-main ()
-{
-return pthread_create ();
- ;
- return 0;
-}
-_ACEOF
-rm -f conftest.$ac_objext conftest$ac_exeext
-if { (ac_try="$ac_link"
-case "(($ac_try" in
- *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
- *) ac_try_echo=$ac_try;;
-esac
-eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
-$as_echo "$ac_try_echo") >&5
- (eval "$ac_link") 2>conftest.er1
- ac_status=$?
- grep -v '^ *+' conftest.er1 >conftest.err
- rm -f conftest.er1
- cat conftest.err >&5
- $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
- (exit $ac_status); } && {
- test -z "$ac_cxx_werror_flag" ||
- test ! -s conftest.err
- } && test -s conftest$ac_exeext && {
- test "$cross_compiling" = yes ||
- $as_test_x conftest$ac_exeext
- }; then
- ac_cv_lib_pthread_pthread_create=yes
-else
- $as_echo "$as_me: failed program was:" >&5
-sed 's/^/| /' conftest.$ac_ext >&5
-
- ac_cv_lib_pthread_pthread_create=no
-fi
-
-rm -rf conftest.dSYM
-rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \
- conftest$ac_exeext conftest.$ac_ext
-LIBS=$ac_check_lib_save_LIBS
-fi
-{ $as_echo "$as_me:$LINENO: result: $ac_cv_lib_pthread_pthread_create" >&5
-$as_echo "$ac_cv_lib_pthread_pthread_create" >&6; }
-if test "x$ac_cv_lib_pthread_pthread_create" = x""yes; then
- cat >>confdefs.h <<_ACEOF
-#define HAVE_LIBPTHREAD 1
-_ACEOF
-
- LIBS="-lpthread $LIBS"
-
-fi
-
-
if test "$found_fftw" = "y"; then
CPPFLAGS="$CPPFLAGS -Dcimg_use_fftw3"
@@ -20627,6 +21271,13 @@ $as_echo "$as_me: error: conditional \"am__fastdepGCJ\" was never defined.
Usually this means the macro was only invoked conditionally." >&2;}
{ (exit 1); exit 1; }; }
fi
+if test -z "${HAVE_PTHREAD_TRUE}" && test -z "${HAVE_PTHREAD_FALSE}"; then
+ { { $as_echo "$as_me:$LINENO: error: conditional \"HAVE_PTHREAD\" was never defined.
+Usually this means the macro was only invoked conditionally." >&5
+$as_echo "$as_me: error: conditional \"HAVE_PTHREAD\" was never defined.
+Usually this means the macro was only invoked conditionally." >&2;}
+ { (exit 1); exit 1; }; }
+fi
if test -z "${WITH_JAVA_TRUE}" && test -z "${WITH_JAVA_FALSE}"; then
{ { $as_echo "$as_me:$LINENO: error: conditional \"WITH_JAVA\" was never defined.
Usually this means the macro was only invoked conditionally." >&5
@@ -20991,7 +21642,7 @@ exec 6>&1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by pHash $as_me 0.8.1, which was
+This file was extended by pHash $as_me 0.9.0, which was
generated by GNU Autoconf 2.63. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -21054,7 +21705,7 @@ Report bugs to <bug-autoconf at gnu.org>."
_ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_version="\\
-pHash config.status 0.8.1
+pHash config.status 0.9.0
configured by $0, generated by GNU Autoconf 2.63,
with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\"
diff --git a/configure.ac b/configure.ac
index 60da316..df9137e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,16 +2,43 @@
# Process this file with autoconf to produce a configure script.
AC_PREREQ(2.61)
-AC_INIT([pHash],[0.8.1],[support at phash.org])
+AC_INIT([pHash],[0.9.0],[support at phash.org])
AM_INIT_AUTOMAKE($PACKAGE_NAME,$PACKAGE_VERSION)
AC_CONFIG_SRCDIR([src/pHash.cpp])
AC_CONFIG_HEADERS([config.h])
# Checks for programs.
+m4_include([m4/ax_pthread.m4])
+
AC_PROG_CXX
AC_PROG_CC
AM_PROG_GCJ
+if test -z $DISTRO; then
+ AC_CHECK_FILE(/etc/gentoo-release, [DISTRO="gentoo"])
+ AC_CHECK_FILE(/etc/redhat-release, [DISTRO="redhat"])
+ AC_CHECK_FILE(/etc/slackware-version, [DISTRO="slackware"])
+ AC_CHECK_FILE(/etc/debian_version, [DISTRO="debian"])
+ AC_CHECK_FILE(/etc/SuSErelease, [DISTRO="suse"])
+fi
+
+
+AC_ARG_ENABLE(pthread, [AS_HELP_STRING([--disable-pthread],
+[pthread support @<:@default=no@:>@])],
+ [PTHREAD="$enableval"],
+ [PTHREAD="yes"])
+
+if test "$PTHREAD" = "yes"; then
+ AX_PTHREAD([
+ LIBS="$PTHREAD_LIBS $LIBS"
+ CPPFLAGS="$CLFAGS $PTHREAD_CFLAGS"
+ CC="$PTHREAD_CC"
+ AC_DEFINE([HAVE_PTHREAD], [1], [configure with pthread support])
+ ])
+fi
+
+AM_CONDITIONAL([HAVE_PTHREAD], [test "$PTHREAD" = "yes"])
+
AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],
[compile with debugging support @<:@default=no@:>@])],
debug=$enableval, debug=no)
@@ -134,7 +161,9 @@ AS_IF([test x"$image_hash" != x"no"],
echo "*** Configuring image hash ***"
echo
AC_CHECK_CIMG()
- CPPFLAGS="$CPPFLAGS -DHAVE_IMAGE_HASH=1"])
+ AC_DEFINE([HAVE_IMAGE_HASH], [1], [configure with image hash])
+ ])
+
AM_CONDITIONAL(HAVE_IMAGE_HASH, test x$image_hash != xno)
AC_ARG_ENABLE([video-hash], [AS_HELP_STRING([--enable-video-hash],[include support for perceptual video hashes @<:@default=yes@:>@])],
@@ -150,7 +179,9 @@ AS_IF([test x"$video_hash" != x"no"],
if test "$found_cimg" != "y"; then
AC_CHECK_CIMG()
fi
- CPPFLAGS="$CPPFLAGS -DHAVE_VIDEO_HASH=1"])
+ AC_DEFINE([HAVE_VIDEO_HASH],[1], [configure with video hash])
+ ])
+
AM_CONDITIONAL(HAVE_VIDEO_HASH, test x$video_hash != xno)
AC_ARG_ENABLE([audio-hash], [AS_HELP_STRING([--enable-audio-hash],[include support for perceptual audio hashes @<:@default=yes@:>@])],
@@ -163,18 +194,28 @@ AS_IF([test x"$audio_hash" != x"no"],
if test "$found_ffmpeg" != "y"; then
AC_CHECK_FFMPEG()
fi
- CPPFLAGS="$CPPFLAGS -DHAVE_AUDIO_HASH=1"])
+ AC_DEFINE([HAVE_AUDIO_HASH],[1], [configure with audio hash])
+ ])
AM_CONDITIONAL(HAVE_AUDIO_HASH, test x$audio_hash != xno)
+if test "$audio_hash" = "yes" -o "$video_hash" = "yes"; then
+case "$DISTRO" in
+ debian)
+ CPPFLAGS="$CPPFLAGS -I/usr/include/ffmpeg"
+ ;;
+ redhat)
+ CPPFLAGS="$CPPFLAGS -I/usr/include/libavcodec -I/usr/include/libavformat -I/usr/include/libswscale"
+ ;;
+esac
+fi
# Checks for libraries.
AC_CHECK_LIB([jpeg], [jpeg_read_header], [use_jpeg="y"])
AC_CHECK_LIB([m], [sqrt])
AC_CHECK_LIB([png], [png_create_read_struct], [use_png="y"])
-AC_CHECK_LIB([pthread], [pthread_create])
AS_IF([test "$found_fftw" = "y"],
[
diff --git a/examples/Makefile.am b/examples/Makefile.am
index c8d6eaa..89b15df 100644
--- a/examples/Makefile.am
+++ b/examples/Makefile.am
@@ -1,14 +1,14 @@
INCLUDES = -I$(top_srcdir)/src
-bin_PROGRAMS = build_mvptree add_mvptree query_mvptree test_texthash test_texthash2
+bin_PROGRAMS = buildmvptreedct addmvptreedct querymvptreedct test_texthash test_texthash2
-build_mvptree_SOURCES = build_mvptree.cpp
-build_mvptree_LDADD = $(top_srcdir)/src/libpHash.la
+buildmvptreedct_SOURCES = buildmvptree_dctimage.cpp
+buildmvptreedct_LDADD = $(top_srcdir)/src/libpHash.la
-add_mvptree_SOURCES = add_mvptree.cpp
-add_mvptree_LDADD = $(top_srcdir)/src/libpHash.la
+addmvptreedct_SOURCES = add_mvptree_dct.cpp
+addmvptreedct_LDADD = $(top_srcdir)/src/libpHash.la
-query_mvptree_SOURCES = query_mvptree.cpp
-query_mvptree_LDADD = $(top_srcdir)/src/libpHash.la
+querymvptreedct_SOURCES = query_mvptree_dct.cpp
+querymvptreedct_LDADD = $(top_srcdir)/src/libpHash.la
test_texthash_SOURCES = test_texthash.cpp
test_texthash_LDADD = $(top_srcdir)/src/libpHash.la
@@ -36,7 +36,6 @@ if HAVE_IMAGE_HASH
bin_PROGRAMS += test_image test_mhimagehash
test_image_SOURCES = test_imagephash.cpp
test_image_LDADD = $(top_srcdir)/src/libpHash.la
-
test_mhimagehash_SOURCES = test_mhimagehash.cpp
test_mhimagehash_LDADD = $(top_srcdir)/src/libpHash.la
endif
@@ -45,6 +44,5 @@ if HAVE_VIDEO_HASH
bin_PROGRAMS += test_video
test_video_SOURCES = test_dctvideohash.cpp
test_video_LDADD = $(top_srcdir)/src/libpHash.la
-
endif
diff --git a/examples/Makefile.in b/examples/Makefile.in
index ca55bb6..dcabce2 100644
--- a/examples/Makefile.in
+++ b/examples/Makefile.in
@@ -34,8 +34,8 @@ PRE_UNINSTALL = :
POST_UNINSTALL = :
build_triplet = @build@
host_triplet = @host@
-bin_PROGRAMS = build_mvptree$(EXEEXT) add_mvptree$(EXEEXT) \
- query_mvptree$(EXEEXT) test_texthash$(EXEEXT) \
+bin_PROGRAMS = buildmvptreedct$(EXEEXT) addmvptreedct$(EXEEXT) \
+ querymvptreedct$(EXEEXT) test_texthash$(EXEEXT) \
test_texthash2$(EXEEXT) $(am__EXEEXT_1) $(am__EXEEXT_2) \
$(am__EXEEXT_3)
@HAVE_AUDIO_HASH_TRUE at am__append_1 = test_audio build_mvptree_audio add_mvptree_audio query_mvptree_audio
@@ -44,7 +44,8 @@ bin_PROGRAMS = build_mvptree$(EXEEXT) add_mvptree$(EXEEXT) \
subdir = examples
DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/configure.ac
+am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \
+ $(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
mkinstalldirs = $(install_sh) -d
@@ -60,33 +61,33 @@ CONFIG_CLEAN_VPATH_FILES =
@HAVE_VIDEO_HASH_TRUE at am__EXEEXT_3 = test_video$(EXEEXT)
am__installdirs = "$(DESTDIR)$(bindir)"
PROGRAMS = $(bin_PROGRAMS)
-am_add_mvptree_OBJECTS = add_mvptree.$(OBJEXT)
-add_mvptree_OBJECTS = $(am_add_mvptree_OBJECTS)
-add_mvptree_DEPENDENCIES = $(top_srcdir)/src/libpHash.la
am__add_mvptree_audio_SOURCES_DIST = add_mvptree_audio.cpp
@HAVE_AUDIO_HASH_TRUE at am_add_mvptree_audio_OBJECTS = \
@HAVE_AUDIO_HASH_TRUE@ add_mvptree_audio.$(OBJEXT)
add_mvptree_audio_OBJECTS = $(am_add_mvptree_audio_OBJECTS)
@HAVE_AUDIO_HASH_TRUE at add_mvptree_audio_DEPENDENCIES = \
@HAVE_AUDIO_HASH_TRUE@ $(top_srcdir)/src/libpHash.la
-am_build_mvptree_OBJECTS = build_mvptree.$(OBJEXT)
-build_mvptree_OBJECTS = $(am_build_mvptree_OBJECTS)
-build_mvptree_DEPENDENCIES = $(top_srcdir)/src/libpHash.la
+am_addmvptreedct_OBJECTS = add_mvptree_dct.$(OBJEXT)
+addmvptreedct_OBJECTS = $(am_addmvptreedct_OBJECTS)
+addmvptreedct_DEPENDENCIES = $(top_srcdir)/src/libpHash.la
am__build_mvptree_audio_SOURCES_DIST = build_mvptree_audio.cpp
@HAVE_AUDIO_HASH_TRUE at am_build_mvptree_audio_OBJECTS = \
@HAVE_AUDIO_HASH_TRUE@ build_mvptree_audio.$(OBJEXT)
build_mvptree_audio_OBJECTS = $(am_build_mvptree_audio_OBJECTS)
@HAVE_AUDIO_HASH_TRUE at build_mvptree_audio_DEPENDENCIES = \
@HAVE_AUDIO_HASH_TRUE@ $(top_srcdir)/src/libpHash.la
-am_query_mvptree_OBJECTS = query_mvptree.$(OBJEXT)
-query_mvptree_OBJECTS = $(am_query_mvptree_OBJECTS)
-query_mvptree_DEPENDENCIES = $(top_srcdir)/src/libpHash.la
+am_buildmvptreedct_OBJECTS = buildmvptree_dctimage.$(OBJEXT)
+buildmvptreedct_OBJECTS = $(am_buildmvptreedct_OBJECTS)
+buildmvptreedct_DEPENDENCIES = $(top_srcdir)/src/libpHash.la
am__query_mvptree_audio_SOURCES_DIST = query_mvptree_audio.cpp
@HAVE_AUDIO_HASH_TRUE at am_query_mvptree_audio_OBJECTS = \
@HAVE_AUDIO_HASH_TRUE@ query_mvptree_audio.$(OBJEXT)
query_mvptree_audio_OBJECTS = $(am_query_mvptree_audio_OBJECTS)
@HAVE_AUDIO_HASH_TRUE at query_mvptree_audio_DEPENDENCIES = \
@HAVE_AUDIO_HASH_TRUE@ $(top_srcdir)/src/libpHash.la
+am_querymvptreedct_OBJECTS = query_mvptree_dct.$(OBJEXT)
+querymvptreedct_OBJECTS = $(am_querymvptreedct_OBJECTS)
+querymvptreedct_DEPENDENCIES = $(top_srcdir)/src/libpHash.la
am__test_audio_SOURCES_DIST = test_audiophash.cpp
@HAVE_AUDIO_HASH_TRUE at am_test_audio_OBJECTS = \
@HAVE_AUDIO_HASH_TRUE@ test_audiophash.$(OBJEXT)
@@ -130,18 +131,19 @@ CXXLD = $(CXX)
CXXLINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
--mode=link $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
-SOURCES = $(add_mvptree_SOURCES) $(add_mvptree_audio_SOURCES) \
- $(build_mvptree_SOURCES) $(build_mvptree_audio_SOURCES) \
- $(query_mvptree_SOURCES) $(query_mvptree_audio_SOURCES) \
+SOURCES = $(add_mvptree_audio_SOURCES) $(addmvptreedct_SOURCES) \
+ $(build_mvptree_audio_SOURCES) $(buildmvptreedct_SOURCES) \
+ $(query_mvptree_audio_SOURCES) $(querymvptreedct_SOURCES) \
$(test_audio_SOURCES) $(test_image_SOURCES) \
$(test_mhimagehash_SOURCES) $(test_texthash_SOURCES) \
$(test_texthash2_SOURCES) $(test_video_SOURCES)
-DIST_SOURCES = $(add_mvptree_SOURCES) \
- $(am__add_mvptree_audio_SOURCES_DIST) $(build_mvptree_SOURCES) \
+DIST_SOURCES = $(am__add_mvptree_audio_SOURCES_DIST) \
+ $(addmvptreedct_SOURCES) \
$(am__build_mvptree_audio_SOURCES_DIST) \
- $(query_mvptree_SOURCES) \
+ $(buildmvptreedct_SOURCES) \
$(am__query_mvptree_audio_SOURCES_DIST) \
- $(am__test_audio_SOURCES_DIST) $(am__test_image_SOURCES_DIST) \
+ $(querymvptreedct_SOURCES) $(am__test_audio_SOURCES_DIST) \
+ $(am__test_image_SOURCES_DIST) \
$(am__test_mhimagehash_SOURCES_DIST) $(test_texthash_SOURCES) \
$(test_texthash2_SOURCES) $(am__test_video_SOURCES_DIST)
ETAGS = etags
@@ -207,6 +209,9 @@ PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
+PTHREAD_CC = @PTHREAD_CC@
+PTHREAD_CFLAGS = @PTHREAD_CFLAGS@
+PTHREAD_LIBS = @PTHREAD_LIBS@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
@@ -226,6 +231,7 @@ am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
+ax_pthread_config = @ax_pthread_config@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
@@ -268,12 +274,12 @@ top_build_prefix = @top_build_prefix@
top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
INCLUDES = -I$(top_srcdir)/src
-build_mvptree_SOURCES = build_mvptree.cpp
-build_mvptree_LDADD = $(top_srcdir)/src/libpHash.la
-add_mvptree_SOURCES = add_mvptree.cpp
-add_mvptree_LDADD = $(top_srcdir)/src/libpHash.la
-query_mvptree_SOURCES = query_mvptree.cpp
-query_mvptree_LDADD = $(top_srcdir)/src/libpHash.la
+buildmvptreedct_SOURCES = buildmvptree_dctimage.cpp
+buildmvptreedct_LDADD = $(top_srcdir)/src/libpHash.la
+addmvptreedct_SOURCES = add_mvptree_dct.cpp
+addmvptreedct_LDADD = $(top_srcdir)/src/libpHash.la
+querymvptreedct_SOURCES = query_mvptree_dct.cpp
+querymvptreedct_LDADD = $(top_srcdir)/src/libpHash.la
test_texthash_SOURCES = test_texthash.cpp
test_texthash_LDADD = $(top_srcdir)/src/libpHash.la
test_texthash2_SOURCES = test_texthash2.cpp
@@ -369,24 +375,24 @@ clean-binPROGRAMS:
list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
echo " rm -f" $$list; \
rm -f $$list
-add_mvptree$(EXEEXT): $(add_mvptree_OBJECTS) $(add_mvptree_DEPENDENCIES)
- @rm -f add_mvptree$(EXEEXT)
- $(CXXLINK) $(add_mvptree_OBJECTS) $(add_mvptree_LDADD) $(LIBS)
add_mvptree_audio$(EXEEXT): $(add_mvptree_audio_OBJECTS) $(add_mvptree_audio_DEPENDENCIES)
@rm -f add_mvptree_audio$(EXEEXT)
$(CXXLINK) $(add_mvptree_audio_OBJECTS) $(add_mvptree_audio_LDADD) $(LIBS)
-build_mvptree$(EXEEXT): $(build_mvptree_OBJECTS) $(build_mvptree_DEPENDENCIES)
- @rm -f build_mvptree$(EXEEXT)
- $(CXXLINK) $(build_mvptree_OBJECTS) $(build_mvptree_LDADD) $(LIBS)
+addmvptreedct$(EXEEXT): $(addmvptreedct_OBJECTS) $(addmvptreedct_DEPENDENCIES)
+ @rm -f addmvptreedct$(EXEEXT)
+ $(CXXLINK) $(addmvptreedct_OBJECTS) $(addmvptreedct_LDADD) $(LIBS)
build_mvptree_audio$(EXEEXT): $(build_mvptree_audio_OBJECTS) $(build_mvptree_audio_DEPENDENCIES)
@rm -f build_mvptree_audio$(EXEEXT)
$(CXXLINK) $(build_mvptree_audio_OBJECTS) $(build_mvptree_audio_LDADD) $(LIBS)
-query_mvptree$(EXEEXT): $(query_mvptree_OBJECTS) $(query_mvptree_DEPENDENCIES)
- @rm -f query_mvptree$(EXEEXT)
- $(CXXLINK) $(query_mvptree_OBJECTS) $(query_mvptree_LDADD) $(LIBS)
+buildmvptreedct$(EXEEXT): $(buildmvptreedct_OBJECTS) $(buildmvptreedct_DEPENDENCIES)
+ @rm -f buildmvptreedct$(EXEEXT)
+ $(CXXLINK) $(buildmvptreedct_OBJECTS) $(buildmvptreedct_LDADD) $(LIBS)
query_mvptree_audio$(EXEEXT): $(query_mvptree_audio_OBJECTS) $(query_mvptree_audio_DEPENDENCIES)
@rm -f query_mvptree_audio$(EXEEXT)
$(CXXLINK) $(query_mvptree_audio_OBJECTS) $(query_mvptree_audio_LDADD) $(LIBS)
+querymvptreedct$(EXEEXT): $(querymvptreedct_OBJECTS) $(querymvptreedct_DEPENDENCIES)
+ @rm -f querymvptreedct$(EXEEXT)
+ $(CXXLINK) $(querymvptreedct_OBJECTS) $(querymvptreedct_LDADD) $(LIBS)
test_audio$(EXEEXT): $(test_audio_OBJECTS) $(test_audio_DEPENDENCIES)
@rm -f test_audio$(EXEEXT)
$(CXXLINK) $(test_audio_OBJECTS) $(test_audio_LDADD) $(LIBS)
@@ -412,12 +418,12 @@ mostlyclean-compile:
distclean-compile:
-rm -f *.tab.c
- at AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/add_mvptree.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/add_mvptree_audio.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/build_mvptree.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/add_mvptree_dct.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/build_mvptree_audio.Po at am__quote@
- at AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/query_mvptree.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/buildmvptree_dctimage.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/query_mvptree_audio.Po at am__quote@
+ at AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/query_mvptree_dct.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/test_audiophash.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/test_dctvideohash.Po at am__quote@
@AMDEP_TRUE@@am__include@ @am__quote at ./$(DEPDIR)/test_imagephash.Po at am__quote@
diff --git a/examples/add_mvptree_audio.cpp b/examples/add_mvptree_audio.cpp
index e83619c..c1da2f9 100644
--- a/examples/add_mvptree_audio.cpp
+++ b/examples/add_mvptree_audio.cpp
@@ -45,6 +45,7 @@ float audiohashdistance(DP *dpA, DP *dpB){
maxC = ptrC[i];
}
double res = 1000*(1-maxC);
+ free(ptrC);
return (float)res;
}
@@ -56,9 +57,9 @@ int main(int argc, char **argv){
const int sr = 8000;
const int nbchannels = 1;
+ const float nbsecs = 45.0f;
MVPFile mvpfile;
- ph_mvp_init(&mvpfile);
mvpfile.filename = strdup(filename);
mvpfile.hashdist = audiohashdistance;
mvpfile.hash_type = UINT32ARRAY;
@@ -68,58 +69,56 @@ int main(int argc, char **argv){
char **files = ph_readfilenames(dir_name,nbfiles);
if (!files){
printf("mem alloc error\n");
- exit(1);
+ return -1;
}
printf("nbfiles = %d\n", nbfiles);
DP **hashlist = (DP**)malloc(nbfiles*sizeof(DP*));
if (!hashlist){
printf("mem alloc error\n");
- exit(1);
+ return -2;
}
int count = 0;
- float *buf;
- int N = 0;
- uint32_t *hash;
- int hashlen = 0;
- DP *dp;
+ float *sigbuf = (float*)malloc(1<<21);
+ int buflen = (1<<21)/sizeof(float);
+
for (int i=0;i<nbfiles;i++){
printf("file[%d]: %s ", i, files[i]);
- buf = ph_readaudio(files[i], sr, nbchannels, N);
+ int N = buflen;
+ float *buf = ph_readaudio(files[i], sr, nbchannels, sigbuf, N, nbsecs);
if (!buf){
printf("unable to read file\n");
continue;
}
- printf("N = %d ", N);
-
- hash = ph_audiohash(buf, N, sr, hashlen);
+ printf("signal length %d\n", N);
+ int nbframes = 0;
+ uint32_t *hash = ph_audiohash(buf, N, sr, nbframes);
if (!hash){
printf("unable to get hash\n");
- free(buf);
+ free(hash);;
+ continue;
}
- printf("nb hashes = %d\n", hashlen);
-
- dp = ph_malloc_datapoint(mvpfile.hash_type,mvpfile.pathlength);
- if (!dp){
+ printf("nbframes %d\n", nbframes);
+ hashlist[count] = ph_malloc_datapoint(mvpfile.hash_type);
+ if (!hashlist[count]){
printf("mem alloc error\n");
- free(buf);
free(hash);
+ return -3;
}
- dp->id = files[i];
- dp->hash = (void*)hash;
- dp->hash_length = hashlen;
+ hashlist[count]->id = files[i];
+ hashlist[count]->hash = (void*)hash;
+ hashlist[count++]->hash_length = nbframes;
- hashlist[count++] = dp;
}
printf("add %d files to file %s\n", count, filename);
- int n = ph_add_mvptree(&mvpfile, hashlist, count);
- if (n <= 0){
- printf("unable to add points to %s\n", filename);
+ int nbsaved=0;
+ MVPRetCode ret = ph_add_mvptree(&mvpfile, hashlist, count,nbsaved);
+ if (ret != PH_SUCCESS){
+ printf("unable to add points to %s - retcode %d\n", filename, ret);
}
- printf("save %d files out of %d\n", n, count);
-
- free(hashlist);
+ printf("save %d files out of %d to %s\n", nbsaved, count, filename);
+ printf("done\n");
return 0;
}
diff --git a/examples/add_mvptree.cpp b/examples/add_mvptree_dct.cpp
similarity index 59%
rename from examples/add_mvptree.cpp
rename to examples/add_mvptree_dct.cpp
index db8d1dd..9a25caa 100644
--- a/examples/add_mvptree.cpp
+++ b/examples/add_mvptree_dct.cpp
@@ -18,7 +18,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
*/
@@ -28,70 +28,81 @@
float distancefunc(DP *pa, DP *pb){
- uint8_t *hashA = (uint8_t*)pa->hash;
- uint8_t *hashB = (uint8_t*)pb->hash;
- float d = 10*ph_hammingdistance2(hashA,pa->hash_length, hashB, pb->hash_length);
- float res = exp(d);
- return res;
+ float d = ph_hamming_distance(*((ulong64*)pa->hash), *((ulong64*)pb->hash));
+ return d;
}
int main(int argc, char **argv){
if (argc < 3){
printf("not enough input args\n");
- return 1;
+ printf("usage: %s directory filename\n", argv[0]);
+ return -1;
}
const char *dir_name = argv[1];/* name of dir to retrieve image files */
const char *filename = argv[2];/* name of file to save db */
- int alpha = 2;
- int lvl = 2;
-
MVPFile mvpfile;
- ph_mvp_init(&mvpfile);
mvpfile.filename = strdup(filename);
mvpfile.hashdist = distancefunc;
- mvpfile.hash_type = BYTEARRAY;
-
+ mvpfile.hash_type = UINT64ARRAY;
int nbfiles = 0;
printf("dir name: %s\n", dir_name);
char **files = ph_readfilenames(dir_name,nbfiles);
if (!files){
printf("mem alloc error\n");
- exit(1);
+ return -2;
}
printf("nbfiles = %d\n", nbfiles);
DP **hashlist = (DP**)malloc(nbfiles*sizeof(DP*));
if (!hashlist){
printf("mem alloc error\n");
- exit(1);
+ return -3;
}
- int hashlength;
+
int count = 0;
+ ulong64 tmphash = 0;
for (int i=0;i<nbfiles;i++){
- printf("file[%d]: %s\n", i, files[i]);
- hashlist[count] = ph_malloc_datapoint(mvpfile.hash_type,mvpfile.pathlength);
+ hashlist[count] = ph_malloc_datapoint(mvpfile.hash_type);
if (hashlist[count] == NULL){
printf("mem alloc error\n");
- continue;
+ return -4;
}
- hashlist[count]->id = files[i];
- hashlist[count]->hash = ph_mh_imagehash(files[i],hashlength,alpha,lvl);
+ hashlist[count]->hash = malloc(sizeof(ulong64));
if (hashlist[count]->hash == NULL){
- printf("unable to get hash\n");
- continue;
+ printf("mem alloc error\n");
+ return -5;
}
- hashlist[count]->hash_length = hashlength;
+ printf("file[%d] = %s\n", i, files[i]);
+ if (ph_dct_imagehash(files[i],tmphash) < 0){
+ printf("unable to get hash\n");
+ free(hashlist[count]->hash);
+ ph_free_datapoint(hashlist[count]);
+ continue;
+ }
+ hashlist[count]->id = files[i];
+ *((ulong64*)hashlist[count]->hash) = tmphash;
+ hashlist[count]->hash_length = 1;
count++;
}
printf("add files to file %s\n", filename);
- int n = ph_add_mvptree(&mvpfile, hashlist, count);
- printf("number saved %d out of %d\n", n,count);
- if (n <= 0){
- printf("unable to add points to %s\n", filename);
+ int nbsaved;
+ MVPRetCode ret = ph_add_mvptree(&mvpfile, hashlist, count,nbsaved);
+ printf("number saved %d out of %d, ret code %d\n", nbsaved,count,ret);
+
+
+ for (int i=0;i<nbfiles;i++){
+ free(files[i]);
+ }
+ free(files);
+
+ for (int i=0;i<nbfiles;i++){
+ free(hashlist[i]->hash);
+ ph_free_datapoint(hashlist[i]);
}
+ free(hashlist);
return 0;
}
diff --git a/examples/build_mvptree_audio.cpp b/examples/build_mvptree_audio.cpp
index ce65e72..b73b95f 100644
--- a/examples/build_mvptree_audio.cpp
+++ b/examples/build_mvptree_audio.cpp
@@ -23,7 +23,6 @@
*/
#include <stdio.h>
-#include <dirent.h>
#include "pHash.h"
#include "audiophash.h"
@@ -39,11 +38,12 @@ float audiohashdistance(DP *dpA, DP *dpB){
int Nc=0;
double *ptrC = ph_audio_distance_ber(hash1, N1, hash2, N2, threshold,blocksize,Nc);
- double maxC = 0;
+ double maxC = 0.0f;
for (int i=0;i<Nc;i++){
if (ptrC[i] > maxC)
maxC = ptrC[i];
}
+ free(ptrC);
double res = 1000*(1-maxC);
return (float)res;
@@ -56,67 +56,73 @@ int main(int argc, char **argv){
MVPFile mvpfile;
mvpfile.branchfactor = 2;
- mvpfile.leafcapacity = 25;
+ mvpfile.leafcapacity = 40;
mvpfile.pathlength = 5;
- mvpfile.pgsize = (1 << 18);
+ mvpfile.pgsize = (1 << 19); /* 524,288 */
mvpfile.filename = strdup(filename);
mvpfile.hashdist = audiohashdistance;
mvpfile.hash_type = UINT32ARRAY;
int sr = 8000;
int nbchannels = 1;
+ float nbsecs = 45.0f;
int nbfiles = 0;
printf("dir name: %s\n", dir_name);
char **files = ph_readfilenames(dir_name,nbfiles);
if (!files){
- printf("mem alloc error\n");
- exit(1);
+ printf("unable to read files from directory %s\n",dir_name);
+ return -2;
}
printf("nbfiles = %d\n", nbfiles);
DP **hashlist = (DP**)malloc(nbfiles*sizeof(DP**));
if (!hashlist){
printf("mem alloc error\n");
- exit(1);
+ return -3;
}
int count = 0;
- float *buf = NULL;
- uint32_t *hash = NULL;
- DP *dp = NULL;
- int nbsamples = 0;
- int hashlength = 0;
+ float *sigbuf = (float*)malloc(1<<21);
+ int buflen = (1<<21)/sizeof(float);
+
for (int i=0;i<nbfiles;i++){
- buf = ph_readaudio(files[i], sr, nbchannels, nbsamples);
+ printf("file[%d]: %s\n", i, files[i]);
+ int N = buflen;
+ float *buf = ph_readaudio(files[i], sr, nbchannels, sigbuf, N, nbsecs);
if (!buf){
printf("unable to read file: %s\n", files[i]);
continue;
}
- hash = ph_audiohash(buf, nbsamples, sr, hashlength);
+ printf("signal length %d\n", N);
+ int nbframes=0;
+ uint32_t *hash = ph_audiohash(buf, N, sr, nbframes);
if (!hash){
- printf("unable to compute hash: %s\n", files[i]);
+ printf("unable to compute hash\n");
continue;
}
- printf("file[%d]: %s, samples %d, hashlen = %d\n", i, files[i],nbsamples,hashlength);
- dp = ph_malloc_datapoint(mvpfile.hash_type,mvpfile.pathlength);
- if (!dp){
+ printf("nbframes %d\n", nbframes);
+ hashlist[count] = ph_malloc_datapoint(mvpfile.hash_type);
+ if (!hashlist[count]){
printf("mem alloc error\n");
- exit(1);
+ free(hash);
+ return -4;
}
- dp->id = files[i];
- dp->hash = hash;
- dp->hash_length = hashlength;
- hashlist[count++] = dp;
- free(buf);
+ hashlist[count]->id = files[i];
+ hashlist[count]->hash = hash;
+ hashlist[count++]->hash_length = nbframes;
+
+ if (buf != sigbuf)
+ free(buf);
}
printf("save %d files\n", count);
- if (ph_save_mvptree(&mvpfile, hashlist, count) < 0){
- printf("unable to save %s\n", filename);
- exit(1);
+ MVPRetCode ret = ph_save_mvptree(&mvpfile, hashlist, count);
+ if (ret != PH_SUCCESS){
+ printf("unable to save %s file - ret code %d\n", filename, ret);
}
+ printf("done\n");
return 0;
}
diff --git a/examples/build_mvptree.cpp b/examples/buildmvptree_dctimage.cpp
similarity index 61%
rename from examples/build_mvptree.cpp
rename to examples/buildmvptree_dctimage.cpp
index e826cec..c44fb4c 100644
--- a/examples/build_mvptree.cpp
+++ b/examples/buildmvptree_dctimage.cpp
@@ -18,84 +18,93 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
*/
-
#include <stdio.h>
#include <math.h>
#include "pHash.h"
float distancefunc(DP *pa, DP *pb){
- if ((!pa)||(!pb)){
- printf("dp is null\n");
- return -1.0;
- }
- uint8_t *hashA = (uint8_t*)pa->hash;
- uint8_t *hashB = (uint8_t*)pb->hash;
- float d = 10*ph_hammingdistance2(hashA,pa->hash_length,hashB,pb->hash_length);
- float res = exp(d);
- return res;
+ float d = ph_hamming_distance(*((ulong64*)pa->hash), *((ulong64*)pb->hash));
+ return d;
}
int main(int argc, char **argv){
if (argc < 3){
printf("not enough input args\n");
- return 1;
+ printf("usage: %s dirname filename\n", argv[0]);
+ return -1;
}
const char *dir_name = argv[1];/* name of dir to retrieve image files */
const char *filename = argv[2];/* name of file to save db */
- int alpha = 2;
- int lvl = 2;
-
MVPFile mvpfile;
- ph_mvp_init(&mvpfile);
+ mvpfile.branchfactor = 2;
+ mvpfile.pathlength = 5;
+ mvpfile.leafcapacity = 50;
+ mvpfile.pgsize = 8192;
mvpfile.filename = strdup(filename);
mvpfile.hashdist = distancefunc;
- mvpfile.hash_type = BYTEARRAY;
-
+ mvpfile.hash_type = UINT64ARRAY;
int nbfiles = 0;
printf("dir name: %s\n", dir_name);
char **files = ph_readfilenames(dir_name,nbfiles);
if (!files){
printf("mem alloc error\n");
- exit(1);
+ return -2;
}
printf("nbfiles = %d\n", nbfiles);
DP **hashlist = (DP**)malloc(nbfiles*sizeof(DP*));
if (!hashlist){
printf("mem alloc error\n");
- exit(1);
+ return -3;
}
- int hashlength;
+
int count = 0;
for (int i=0;i<nbfiles;i++){
- printf("file[%d]: %s\n", i, files[i]);
- hashlist[count] = ph_malloc_datapoint(mvpfile.hash_type,mvpfile.pathlength);
+ hashlist[count] = ph_malloc_datapoint(mvpfile.hash_type);
if (hashlist[count] == NULL){
printf("mem alloc error\n");
- exit(1);
+ return -4;
}
- hashlist[count]->id = files[i];
- hashlist[count]->hash = ph_mh_imagehash(files[i],hashlength, alpha, lvl);
+ hashlist[count]->hash = malloc(sizeof(ulong64));
if (hashlist[count]->hash == NULL){
+ printf("mem alloc error\n");
+ return -5;
+ }
+
+ if (ph_dct_imagehash(files[i],*((ulong64*)hashlist[count]->hash)) < 0){
printf("unable to get hash\n");
- exit(1);
+ continue;
}
- printf("len %d\n", hashlength);
- hashlist[count]->hash_length = hashlength;
+
+ printf("files[%d]: %s hash = %llx\n", i, files[i], *((ulong64*)hashlist[count]->hash));
+
+ hashlist[count]->id = files[i];
+ hashlist[count]->hash_length = 1;
count++;
}
+
+ MVPRetCode ret = ph_save_mvptree(&mvpfile, hashlist, count);
+ printf("save: ret code %d\n", ret);
+
+
- if (ph_save_mvptree(&mvpfile, hashlist, count) < 0){
- printf("unable to save %s\n", filename);
- exit(1);
+ for (int i=0;i<nbfiles;i++){
+ free(files[i]);
+ }
+ free(files);
+
+ for (int i=0;i<nbfiles;i++){
+ free(hashlist[i]->hash);
+ ph_free_datapoint(hashlist[i]);
}
+ free(hashlist);
return 0;
}
diff --git a/examples/query_mvptree.cpp b/examples/query_mvptree.cpp
deleted file mode 100644
index 09c5358..0000000
--- a/examples/query_mvptree.cpp
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
-
- pHash, the open source perceptual hash library
- Copyright (C) 2009 Aetilius, Inc.
- All rights reserved.
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-
- Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
-
-*/
-
-#include <stdio.h>
-#include <math.h>
-#include "pHash.h"
-
-static int nb_calcs;
-
-
-float distancefunc(DP *pa, DP *pb){
- nb_calcs++;
- uint8_t *hashA = (uint8_t*)pa->hash;
- uint8_t *hashB = (uint8_t*)pb->hash;
- float d = 10*ph_hammingdistance2(hashA, pa->hash_length, hashB, pb->hash_length);
- float result = exp(d);
- return result;
-
-}
-
-
-int main(int argc, char **argv){
- if (argc <= 3){
- printf("not enough input args\n");
- return 1;
- }
-
- const char *dir_name = argv[1];/* name of files in directory of query images */
- const char *filename = argv[2];/* name of file to save db */
-
- int alpha = 2;
- int lvl = 2;
-
- MVPFile mvpfile;
- ph_mvp_init(&mvpfile);
- mvpfile.filename = strdup(filename);
- mvpfile.hashdist = distancefunc;
- mvpfile.hash_type = BYTEARRAY;
-
- int nbfiles = 0;
- printf("using db %s\n", filename);
- printf("using dir %s for query files\n", dir_name);
- char **files = ph_readfilenames(dir_name,nbfiles);
- if (!files){
- printf("mem alloc error\n");
- exit(1);
- }
-
- printf("nb query files = %d\n", nbfiles);
-
- DP *query = ph_malloc_datapoint(mvpfile.hash_type,mvpfile.pathlength);
-
- float radius;
- if (argc <= 4){
- char *radius_str = argv[3];
- radius = atof(radius_str);
- } else {
- radius = 30.0f;
- }
- printf("radius = %f\n", radius);
-
- const int knearest = 20;
- DP **results = (DP**)malloc(knearest*sizeof(DP**));
- int nbfound = 0, count = 0, sum_calcs = 0;
- int hashlength;
- for (int i=0;i<nbfiles;i++){
- printf("query[%d]: %s\n", i, files[i]);
-
- query->id = files[i];
- query->hash = ph_mh_imagehash(files[i],hashlength,alpha,lvl);
- query->hash_length = hashlength;
-
- printf("do query ...\n");
- nb_calcs = 0;
- nbfound = 0;
- int res = ph_query_mvptree(&mvpfile,query,knearest,radius,results,&nbfound);
- if (res != 0){
- printf("could not complete query\n");
- continue;
- }
- count++;
- sum_calcs += nb_calcs;
-
- printf(" %d files found\n", nbfound);
- for (int i=0;i<nbfound;i++){
- printf(" %d %s\n", i, results[i]->id);
- }
- printf("nb distance calcs: %d\n", nb_calcs);
- free(query->hash);
- }
- float ave_calcs = (float)sum_calcs/(float)count;
- printf("ave calcs/query: %f\n", ave_calcs);
-
-
- return 0;
-}
diff --git a/examples/query_mvptree_audio.cpp b/examples/query_mvptree_audio.cpp
index a65bdf9..5af2e30 100644
--- a/examples/query_mvptree_audio.cpp
+++ b/examples/query_mvptree_audio.cpp
@@ -49,20 +49,25 @@ float audiohashdistance(DP *dpA, DP *dpB){
maxC = ptrC[i];
}
double res = 1000*(1-maxC);
+ free(ptrC);
return (float)res;
}
int main(int argc, char **argv){
-
+ if (argc < 3){
+ printf("not enough input args\n");
+ printf("usage: %s directory dbname radius knearest threshold\n", argv[0]);
+ return -1;
+ }
const char *dir_name = argv[1];/* name of files in directory of query images */
const char *filename = argv[2];/* name of file to save db */
const int sr = 8000;
const int nbchannels = 1;
+ const float nbsecs = 45.0f;
MVPFile mvpfile;
- ph_mvp_init(&mvpfile);
mvpfile.filename = strdup(filename);
mvpfile.hashdist = audiohashdistance;
mvpfile.hash_type = UINT32ARRAY;
@@ -73,64 +78,81 @@ int main(int argc, char **argv){
char **files = ph_readfilenames(dir_name,nbfiles);
if (!files){
printf("mem alloc error\n");
- exit(1);
+ return -2;
}
+ printf("nb files = %d\n", nbfiles);
- printf("nb query files = %d\n", nbfiles);
+ float radius = 700.0;
+ if (argc >= 3){
+ radius = atof(argv[3]);
+ }
+ int knearest = 20;
+ if (argc >= 4){
+ knearest = atoi(argv[4]);
+ }
+ float threshold = 300.0;
+ if (argc >= 5){
+ threshold = atof(argv[5]);
+ }
+ printf("radius %f, knearest %d, threshold %f\n", radius, knearest, threshold);
- DP *query = NULL;
- float radius = 300.0;
- const int knearest = 20;
- DP **results = (DP**)malloc(knearest * sizeof(DP**));
+ DP **results = (DP**)malloc(knearest*sizeof(DP*));
if (!results){
printf("mem alloc error\n");
- exit(1);
+ return -3;
}
+ DP *query = ph_malloc_datapoint(mvpfile.hash_type);
+
int nbfound = 0, count = 0, sum_calcs = 0;
- float *buf;
- int N = 0;
- uint32_t *hash;
- int hashlen = 0;
+ float *sigbuf = (float*)malloc(1<<21);
+ int buflen = (1<<21)/sizeof(float);
for (int i=0;i<nbfiles;i++){
- buf = ph_readaudio(files[i], sr, nbchannels, N);
+ printf("query[%d]: %s\n", i, files[i]);
+ int N = buflen;
+ float *buf = ph_readaudio(files[i], sr, nbchannels, sigbuf, N, nbsecs);
if (!buf){
printf("could not read audio\n");
continue;
}
- hash = ph_audiohash(buf,N,sr,hashlen);
+ printf("samples %d\n", N);
+ int nbframes = 0;
+ uint32_t *hash = ph_audiohash(buf,N,sr,nbframes);
if (!hash){
printf("could not get hash\n");
- free(buf);
continue;
}
- query = ph_malloc_datapoint(mvpfile.hash_type,mvpfile.pathlength);
- if (!query){
- printf("mem alloc error\n");
- free(buf);
- free(hash);
- break;
- }
- query->id = strdup(files[i]);
+ printf("number hash frames %d\n\n", nbframes);
+
+ query->id = files[i];
query->hash = hash;
- query->hash_length = hashlen;
+ query->hash_length = nbframes;
count++;
nb_calcs = 0;
nbfound = 0;
- int res = ph_query_mvptree(&mvpfile,query,knearest,radius,results,&nbfound);
- if (res != 0){
- printf("could not complete query\n");
+ int ret = ph_query_mvptree(&mvpfile,query,knearest,radius,threshold, results,nbfound);
+ if (ret != PH_SUCCESS && ret != PH_ERRCAP){
+ printf("could not complete query - retcode %d\n",ret);
+ free(hash);
continue;
}
sum_calcs += nb_calcs;
- printf("search: %s nbfound: %d\n", files[i],nbfound);
- for (int i=0;i<nbfound;i++){
- printf(" %d %s\n", i, results[i]->id);
+ printf("search: %d found, %d distance calc's\n", nbfound,nb_calcs);
+ for (int j=0;j<nbfound;j++){
+ float dist = audiohashdistance(query,results[j]);
+ printf(" %d %s distance = %f\n", j, results[j]->id,dist);
+ }
+ free(hash);
+ if (sigbuf != buf){
+ free(buf);
+ }
+ for (int j=0;j<nbfound;j++){
+ free(results[j]->id);
+ free(results[j]->hash);
+ ph_free_datapoint(results[j]);
}
- free(query);
- free(buf);
- free(hash);
+ printf("\n\n**********************************************\n");
}
float ave_calcs = (float)sum_calcs/(float)count;
printf("ave calcs/query: %f\n", ave_calcs);
diff --git a/examples/query_mvptree_dct.cpp b/examples/query_mvptree_dct.cpp
new file mode 100644
index 0000000..831b5dc
--- /dev/null
+++ b/examples/query_mvptree_dct.cpp
@@ -0,0 +1,143 @@
+/*
+
+ pHash, the open source perceptual hash library
+ Copyright (C) 2009 Aetilius, Inc.
+ All rights reserved.
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+ Evan Klinger - eklinger at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
+
+*/
+
+#include <stdio.h>
+#include <math.h>
+#include "pHash.h"
+
+static int nb_calcs;
+
+float distancefunc(DP *pa, DP *pb){
+ nb_calcs++;
+ float d = ph_hamming_distance(*((ulong64*)pa->hash),*((ulong64*)pb->hash));
+ return d;
+}
+
+int main(int argc, char **argv){
+ if (argc < 3){
+ printf("not enough input args\n");
+ printf("usage: %s directory dbname [radius] [knearest] [threshold]\n", argv[0]);
+ return -1;
+ }
+
+ const char *dir_name = argv[1];/* name of files in directory of query images */
+ const char *filename = argv[2];/* name of file to save db */
+
+ MVPFile mvpfile;
+ mvpfile.filename = strdup(filename);
+ mvpfile.hashdist = distancefunc;
+ mvpfile.hash_type = UINT64ARRAY;
+
+ int nbfiles = 0;
+ printf("using db %s\n", filename);
+ printf("using dir %s for query files\n", dir_name);
+ char **files = ph_readfilenames(dir_name,nbfiles);
+ if (!files){
+ printf("unable to read files from directory\n");
+ return -2;
+ }
+
+ printf("nb query files = %d\n", nbfiles);
+
+ DP *query = ph_malloc_datapoint(mvpfile.hash_type);
+ if (query == NULL){
+ printf("mem alloc error\n");
+ return -3;
+ }
+
+ float radius = 30.0f;
+ float threshold = 15.0f;
+ int knearest = 20;
+ if (argc >= 4){
+ radius = atof(argv[3]);
+ }
+ if (argc >= 5){
+ knearest = atoi(argv[4]);
+ }
+ if (argc >= 6){
+ threshold = atof(argv[5]);
+ }
+ printf("radius = %f\n", radius);
+ printf("knearest = %d\n", knearest);
+ printf("threshold = %f\n", threshold);
+
+ DP **results = (DP**)malloc(knearest*sizeof(DP*));
+ if (results == NULL){
+ return -3;
+ }
+ ulong64 tmphash = 0x0000000000000000;
+ int nbfound = 0, count = 0, sum_calcs = 0;
+ for (int i=0;i<nbfiles;i++){
+
+ if (ph_dct_imagehash(files[i],tmphash) < 0){
+ printf("unable to get hash\n");
+ continue;
+ }
+ printf("query[%d]: %s %llx\n", i, files[i], tmphash);
+ query->id = files[i];
+ query->hash = &tmphash;
+ query->hash_length = 1;
+
+ nb_calcs = 0;
+ nbfound = 0;
+ MVPRetCode retcode = ph_query_mvptree(&mvpfile,query,knearest,radius,threshold,results,nbfound);
+ if (retcode != PH_SUCCESS && retcode != PH_ERRCAP){
+ printf("could not complete query, %d\n",retcode);
+ continue;
+ }
+ count++;
+ sum_calcs += nb_calcs;
+
+ printf(" %d files found\n", nbfound);
+ for (int j=0;j<nbfound;j++){
+ float d = distancefunc(query, results[j]);
+ printf(" %d %s distance = %f\n", j, results[j]->id, d);
+ }
+ printf("nb distance calcs: %d\n", nb_calcs);
+ for (int j=0;j<nbfound;j++){
+ free(results[j]->id);
+ results[j]->id = NULL;
+ free(results[j]->hash);
+ results[j]->id = NULL;
+ ph_free_datapoint(results[j]);
+ }
+
+ }
+
+ float ave_calcs = (float)sum_calcs/(float)count;
+ printf("ave calcs/query: %f\n", ave_calcs);
+
+
+ for (int i=0;i<nbfiles;i++){
+ free(files[i]);
+ }
+ free(files);
+
+ ph_free_datapoint(query);
+ free(results);
+ free(mvpfile.filename);
+
+ return 0;
+}
+
diff --git a/examples/test_audiophash.cpp b/examples/test_audiophash.cpp
index f5ac90d..6544b2d 100644
--- a/examples/test_audiophash.cpp
+++ b/examples/test_audiophash.cpp
@@ -91,7 +91,7 @@ int main(int argc, char **argv){
printf("***************\n");
for (index=0;index<nbfiles1;index++){
printf("file1: %s\n", files1[index]);
- buf = ph_readaudio(files1[index], sr, channels, buflen);
+ buf = ph_readaudio(files1[index], sr, channels, NULL, buflen);
if (!buf){
printf("unable to read audio\n");
continue;
@@ -106,7 +106,7 @@ int main(int argc, char **argv){
free(buf);
printf("file2: %s\n", files2[index]);
- buf = ph_readaudio(files2[index], sr, channels, buflen);
+ buf = ph_readaudio(files2[index], sr, channels, NULL, buflen);
if (!buf) {
printf("unable to get audio\n");
continue;
diff --git a/examples/test_dctvideohash.cpp b/examples/test_dctvideohash.cpp
index 76f4ddf..f9009a7 100644
--- a/examples/test_dctvideohash.cpp
+++ b/examples/test_dctvideohash.cpp
@@ -82,7 +82,7 @@ int main(int argc, char **argv){
free(hash2);
hash1 = NULL;
hash2 = NULL;
-
+ free(file2);
printf("done\n");
return 0;
}
diff --git a/m4/ax_pthread.m4 b/m4/ax_pthread.m4
new file mode 100644
index 0000000..77ae09c
--- /dev/null
+++ b/m4/ax_pthread.m4
@@ -0,0 +1,283 @@
+# ===========================================================================
+# http://www.nongnu.org/autoconf-archive/ax_pthread.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]])
+#
+# DESCRIPTION
+#
+# This macro figures out how to build C programs using POSIX threads. It
+# sets the PTHREAD_LIBS output variable to the threads library and linker
+# flags, and the PTHREAD_CFLAGS output variable to any special C compiler
+# flags that are needed. (The user can also force certain compiler
+# flags/libs to be tested by setting these environment variables.)
+#
+# Also sets PTHREAD_CC to any special C compiler that is needed for
+# multi-threaded programs (defaults to the value of CC otherwise). (This
+# is necessary on AIX to use the special cc_r compiler alias.)
+#
+# NOTE: You are assumed to not only compile your program with these flags,
+# but also link it with them as well. e.g. you should link with
+# $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS
+#
+# If you are only building threads programs, you may wish to use these
+# variables in your default LIBS, CFLAGS, and CC:
+#
+# LIBS="$PTHREAD_LIBS $LIBS"
+# CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+# CC="$PTHREAD_CC"
+#
+# In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute constant
+# has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to that name
+# (e.g. PTHREAD_CREATE_UNDETACHED on AIX).
+#
+# ACTION-IF-FOUND is a list of shell commands to run if a threads library
+# is found, and ACTION-IF-NOT-FOUND is a list of commands to run it if it
+# is not found. If ACTION-IF-FOUND is not specified, the default action
+# will define HAVE_PTHREAD.
+#
+# Please let the authors know if this macro fails on any platform, or if
+# you have any other suggestions or comments. This macro was based on work
+# by SGJ on autoconf scripts for FFTW (http://www.fftw.org/) (with help
+# from M. Frigo), as well as ac_pthread and hb_pthread macros posted by
+# Alejandro Forero Cuervo to the autoconf macro repository. We are also
+# grateful for the helpful feedback of numerous users.
+#
+# LICENSE
+#
+# Copyright (c) 2008 Steven G. Johnson <stevenj at alum.mit.edu>
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# As a special exception, the respective Autoconf Macro's copyright owner
+# gives unlimited permission to copy, distribute and modify the configure
+# scripts that are the output of Autoconf when processing the Macro. You
+# need not follow the terms of the GNU General Public License when using
+# or distributing such scripts, even though portions of the text of the
+# Macro appear in them. The GNU General Public License (GPL) does govern
+# all other use of the material that constitutes the Autoconf Macro.
+#
+# This special exception to the GPL applies to versions of the Autoconf
+# Macro released by the Autoconf Archive. When you make and distribute a
+# modified version of the Autoconf Macro, you may extend this special
+# exception to the GPL to apply to your modified version as well.
+
+#serial 6
+
+AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD])
+AC_DEFUN([AX_PTHREAD], [
+AC_REQUIRE([AC_CANONICAL_HOST])
+AC_LANG_SAVE
+AC_LANG_C
+ax_pthread_ok=no
+
+# We used to check for pthread.h first, but this fails if pthread.h
+# requires special compiler flags (e.g. on True64 or Sequent).
+# It gets checked for in the link test anyway.
+
+# First of all, check if the user has set any of the PTHREAD_LIBS,
+# etcetera environment variables, and if threads linking works using
+# them:
+if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then
+ save_CFLAGS="$CFLAGS"
+ CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+ save_LIBS="$LIBS"
+ LIBS="$PTHREAD_LIBS $LIBS"
+ AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS])
+ AC_TRY_LINK_FUNC(pthread_join, ax_pthread_ok=yes)
+ AC_MSG_RESULT($ax_pthread_ok)
+ if test x"$ax_pthread_ok" = xno; then
+ PTHREAD_LIBS=""
+ PTHREAD_CFLAGS=""
+ fi
+ LIBS="$save_LIBS"
+ CFLAGS="$save_CFLAGS"
+fi
+
+# We must check for the threads library under a number of different
+# names; the ordering is very important because some systems
+# (e.g. DEC) have both -lpthread and -lpthreads, where one of the
+# libraries is broken (non-POSIX).
+
+# Create a list of thread flags to try. Items starting with a "-" are
+# C compiler flags, and other items are library names, except for "none"
+# which indicates that we try without any flags at all, and "pthread-config"
+# which is a program returning the flags for the Pth emulation library.
+
+ax_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config"
+
+# The ordering *is* (sometimes) important. Some notes on the
+# individual items follow:
+
+# pthreads: AIX (must check this before -lpthread)
+# none: in case threads are in libc; should be tried before -Kthread and
+# other compiler flags to prevent continual compiler warnings
+# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h)
+# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able)
+# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread)
+# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads)
+# -pthreads: Solaris/gcc
+# -mthreads: Mingw32/gcc, Lynx/gcc
+# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it
+# doesn't hurt to check since this sometimes defines pthreads too;
+# also defines -D_REENTRANT)
+# ... -mt is also the pthreads flag for HP/aCC
+# pthread: Linux, etcetera
+# --thread-safe: KAI C++
+# pthread-config: use pthread-config program (for GNU Pth library)
+
+case "${host_cpu}-${host_os}" in
+ *solaris*)
+
+ # On Solaris (at least, for some versions), libc contains stubbed
+ # (non-functional) versions of the pthreads routines, so link-based
+ # tests will erroneously succeed. (We need to link with -pthreads/-mt/
+ # -lpthread.) (The stubs are missing pthread_cleanup_push, or rather
+ # a function called by this macro, so we could check for that, but
+ # who knows whether they'll stub that too in a future libc.) So,
+ # we'll just look for -pthreads and -lpthread first:
+
+ ax_pthread_flags="-pthreads pthread -mt -pthread $ax_pthread_flags"
+ ;;
+
+ *-darwin*)
+ acx_pthread_flags="-pthread $acx_pthread_flags"
+ ;;
+esac
+
+if test x"$ax_pthread_ok" = xno; then
+for flag in $ax_pthread_flags; do
+
+ case $flag in
+ none)
+ AC_MSG_CHECKING([whether pthreads work without any flags])
+ ;;
+
+ -*)
+ AC_MSG_CHECKING([whether pthreads work with $flag])
+ PTHREAD_CFLAGS="$flag"
+ ;;
+
+ pthread-config)
+ AC_CHECK_PROG(ax_pthread_config, pthread-config, yes, no)
+ if test x"$ax_pthread_config" = xno; then continue; fi
+ PTHREAD_CFLAGS="`pthread-config --cflags`"
+ PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`"
+ ;;
+
+ *)
+ AC_MSG_CHECKING([for the pthreads library -l$flag])
+ PTHREAD_LIBS="-l$flag"
+ ;;
+ esac
+
+ save_LIBS="$LIBS"
+ save_CFLAGS="$CFLAGS"
+ LIBS="$PTHREAD_LIBS $LIBS"
+ CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+
+ # Check for various functions. We must include pthread.h,
+ # since some functions may be macros. (On the Sequent, we
+ # need a special flag -Kthread to make this header compile.)
+ # We check for pthread_join because it is in -lpthread on IRIX
+ # while pthread_create is in libc. We check for pthread_attr_init
+ # due to DEC craziness with -lpthreads. We check for
+ # pthread_cleanup_push because it is one of the few pthread
+ # functions on Solaris that doesn't have a non-functional libc stub.
+ # We try pthread_create on general principles.
+ AC_TRY_LINK([#include <pthread.h>
+ static void routine(void* a) {a=0;}
+ static void* start_routine(void* a) {return a;}],
+ [pthread_t th; pthread_attr_t attr;
+ pthread_join(th, 0);
+ pthread_attr_init(&attr);
+ pthread_cleanup_push(routine, 0);
+ pthread_create(&th,0,start_routine,0);
+ pthread_cleanup_pop(0); ],
+ [ax_pthread_ok=yes])
+
+ LIBS="$save_LIBS"
+ CFLAGS="$save_CFLAGS"
+
+ AC_MSG_RESULT($ax_pthread_ok)
+ if test "x$ax_pthread_ok" = xyes; then
+ break;
+ fi
+
+ PTHREAD_LIBS=""
+ PTHREAD_CFLAGS=""
+done
+fi
+
+# Various other checks:
+if test "x$ax_pthread_ok" = xyes; then
+ save_LIBS="$LIBS"
+ LIBS="$PTHREAD_LIBS $LIBS"
+ save_CFLAGS="$CFLAGS"
+ CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+
+ # Detect AIX lossage: JOINABLE attribute is called UNDETACHED.
+ AC_MSG_CHECKING([for joinable pthread attribute])
+ attr_name=unknown
+ for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do
+ AC_TRY_LINK([#include <pthread.h>], [int attr=$attr; return attr;],
+ [attr_name=$attr; break])
+ done
+ AC_MSG_RESULT($attr_name)
+ if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then
+ AC_DEFINE_UNQUOTED(PTHREAD_CREATE_JOINABLE, $attr_name,
+ [Define to necessary symbol if this constant
+ uses a non-standard name on your system.])
+ fi
+
+ AC_MSG_CHECKING([if more special flags are required for pthreads])
+ flag=no
+ case "${host_cpu}-${host_os}" in
+ *-aix* | *-freebsd* | *-darwin*) flag="-D_THREAD_SAFE";;
+ *solaris* | *-osf* | *-hpux*) flag="-D_REENTRANT";;
+ esac
+ AC_MSG_RESULT(${flag})
+ if test "x$flag" != xno; then
+ PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS"
+ fi
+
+ LIBS="$save_LIBS"
+ CFLAGS="$save_CFLAGS"
+
+ # More AIX lossage: must compile with xlc_r or cc_r
+ if test x"$GCC" != xyes; then
+ AC_CHECK_PROGS(PTHREAD_CC, xlc_r cc_r, ${CC})
+ else
+ PTHREAD_CC=$CC
+ fi
+else
+ PTHREAD_CC="$CC"
+fi
+
+AC_SUBST(PTHREAD_LIBS)
+AC_SUBST(PTHREAD_CFLAGS)
+AC_SUBST(PTHREAD_CC)
+
+# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND:
+if test x"$ax_pthread_ok" = xyes; then
+ ifelse([$1],,AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]),[$1])
+ :
+else
+ ax_pthread_ok=no
+ $2
+fi
+AC_LANG_RESTORE
+])dnl AX_PTHREAD
diff --git a/src/Makefile.in b/src/Makefile.in
index ad18687..7ca071f 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -42,7 +42,8 @@ subdir = src
DIST_COMMON = $(am__include_HEADERS_DIST) $(srcdir)/Makefile.am \
$(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/configure.ac
+am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \
+ $(top_srcdir)/configure.ac
am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
$(ACLOCAL_M4)
mkinstalldirs = $(install_sh) -d
@@ -171,6 +172,9 @@ PACKAGE_STRING = @PACKAGE_STRING@
PACKAGE_TARNAME = @PACKAGE_TARNAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
PATH_SEPARATOR = @PATH_SEPARATOR@
+PTHREAD_CC = @PTHREAD_CC@
+PTHREAD_CFLAGS = @PTHREAD_CFLAGS@
+PTHREAD_LIBS = @PTHREAD_LIBS@
RANLIB = @RANLIB@
SED = @SED@
SET_MAKE = @SET_MAKE@
@@ -190,6 +194,7 @@ am__leading_dot = @am__leading_dot@
am__quote = @am__quote@
am__tar = @am__tar@
am__untar = @am__untar@
+ax_pthread_config = @ax_pthread_config@
bindir = @bindir@
build = @build@
build_alias = @build_alias@
diff --git a/src/audiophash.cpp b/src/audiophash.cpp
index 369bdfb..d7a3125 100644
--- a/src/audiophash.cpp
+++ b/src/audiophash.cpp
@@ -106,26 +106,23 @@ int ph_count_samples(const char *filename, int sr,int channels){
return count;
}
-float* ph_readaudio(const char *filename, int sr, int channels, int &N)
+float* ph_readaudio(const char *filename, int sr, int channels, float *sigbuf, int &buflen, const float nbsecs)
{
- N = 500000;
- int cap = 0;
- float *buf = (float*)malloc(N*sizeof(float));
-
- av_log_set_level(AV_LOG_QUIET);
+ av_log_set_level(AV_LOG_QUIET);
av_register_all();
AVFormatContext *pFormatCtx;
// Open file
if(av_open_input_file(&pFormatCtx, filename, NULL, 0, NULL)!=0){
- N=0;
- return NULL ; // Couldn't open file
+ buflen=0;
+ return NULL ; // Couldn't open file
}
// Retrieve stream information
if(av_find_stream_info(pFormatCtx)<0){
- N=0;
+ buflen=0;
+ av_close_input_file(pFormatCtx);
return NULL; // Couldn't find stream information
}
@@ -146,73 +143,89 @@ float* ph_readaudio(const char *filename, int sr, int channels, int &N)
}
}
if(audioStream==-1){
- N = 0;
+ buflen = 0;
+ av_close_input_file(pFormatCtx);
return NULL; //no video stream
}
// Get a pointer to the codec context for the audio stream
pCodecCtx=pFormatCtx->streams[audioStream]->codec;
+ SampleFormat src_fmt = SAMPLE_FMT_S16;
+ SampleFormat dst_fmt = SAMPLE_FMT_S16;
int src_sr = pCodecCtx->sample_rate;
int src_channels = pCodecCtx->channels;
-
+ int64_t totalduration = pFormatCtx->streams[audioStream]->duration;
+ int64_t duration = (nbsecs > 0.0f) ? (int64_t)(sr*nbsecs) : totalduration;
+ duration = (duration <= totalduration) ? duration : totalduration;
+ float *buf = NULL;
+ if (!sigbuf || buflen <= duration){
+ buf = (float*)malloc(duration*sizeof(float)); /* alloc new buffer */
+ buflen = duration;
+ } else {
+ buf = sigbuf; /* use buffer handed in param */
+ }
AVCodec *pCodec;
// Find the decoder
pCodec=avcodec_find_decoder(pCodecCtx->codec_id);
if(pCodec==NULL)
{
- N=0;
+ buflen=0;
+ av_close_input_file(pFormatCtx);
return NULL ; // Codec not found
}
// Open codec
if(avcodec_open(pCodecCtx, pCodec)<0){
- N=0;
+ buflen=0;
+ av_close_input_file(pFormatCtx);
return NULL; // Could not open codec
}
- uint8_t *in_buf = (uint8_t*)malloc(AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE);
+ uint8_t *in_buf = (uint8_t*)av_malloc(AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE);
int in_buf_used, numbytesread, buf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE;
- uint8_t *out_buf = (uint8_t*)malloc(AVCODEC_MAX_AUDIO_FRAME_SIZE);
+ uint8_t *out_buf = (uint8_t*)av_malloc(AVCODEC_MAX_AUDIO_FRAME_SIZE);
int16_t *out_buf16 = (int16_t*)out_buf;
ReSampleContext *rs_ctx = audio_resample_init(channels, src_channels, sr, src_sr);
- int index = 0;
- AVPacket packet;
- while(av_read_frame(pFormatCtx, &packet)>=0)
+ int64_t index = 0;
+ AVPacket *packet = (AVPacket*)malloc(sizeof(AVPacket));
+ av_init_packet(packet);
+ while(av_read_frame(pFormatCtx, packet)>=0 && index < duration)
{
-
- in_buf_used = buf_size;
- numbytesread = avcodec_decode_audio2(pCodecCtx,(int16_t*)in_buf,&in_buf_used,packet.data,packet.size);
- if (numbytesread <= 0){
- continue;
- }
- int cnt = audio_resample(rs_ctx,(short*)out_buf,(short*)in_buf,(int)(in_buf_used/sizeof(int16_t)));
-
- if (cap + cnt > N){
- float *tmpbuf = (float*)realloc(buf,(N + 2*cnt)*sizeof(float));
- if (!tmpbuf){
- N=0;
- return NULL;
+ while (packet->size > 0){
+ in_buf_used = buf_size;
+ numbytesread = avcodec_decode_audio2(pCodecCtx,(int16_t*)in_buf,&in_buf_used,packet->data,packet->size);
+ if (numbytesread < 0){
+ buflen = 0;
+ if (buf != sigbuf) free(buf);
+ buf = NULL;
+ goto audio_cleanup;
}
- buf = tmpbuf;
- N += 2*cnt;
- }
- for (int i=0;i<cnt;i++){
- buf[index+i] = ((float)out_buf16[i]/(float)SHRT_MAX);
-
- }
-
- cap += cnt;
- index += cnt;
+ if (in_buf_used > 0){
+ int cnt = audio_resample(rs_ctx,(short*)out_buf,(short*)in_buf,(int)(in_buf_used/sizeof(int16_t)));
+ if (index + cnt > duration){
+ goto audio_cleanup;
+ }
+ for (int i=0;i<cnt;i++){
+ buf[index+i] = ((float)out_buf16[i]/(float)SHRT_MAX);
+ }
+ index += cnt;
+ }
+ packet->size -= numbytesread;
+ packet->data += numbytesread;
+ }
+ av_destruct_packet_nofree(packet);
}
- free(in_buf);
- free(out_buf);
+ audio_cleanup:
+ free(packet);
+ av_free(in_buf);
+ av_free(out_buf);
audio_resample_close(rs_ctx);
avcodec_close(pCodecCtx);
av_close_input_file(pFormatCtx);
- N = cap;
+ buflen = (int)index;
return buf;
}
@@ -453,3 +466,77 @@ double* ph_audio_distance_ber(uint32_t *hash_a , const int Na, uint32_t *hash_b,
free(dist);
return pC;
}
+#ifdef HAVE_PTHREAD
+
+void *ph_audio_thread(void *p)
+{
+ slice *s = (slice *)p;
+ for(int i = 0; i < s->n; ++i)
+ {
+ DP *dp = (DP *)s->hash_p[i];
+ int N, count;
+ pair<int,int> *p = (pair<int,int> *)s->hash_params;
+ float *buf = ph_readaudio(dp->id, p->first, p->second, N);
+ uint32_t *hash = ph_audiohash(buf, N, p->first, count);
+ free(buf);
+ buf = NULL;
+ dp->hash = hash;
+ dp->hash_length = count;
+ }
+}
+
+DP** ph_audio_hashes(char *files[], int count, int sr, int channels, int threads)
+{
+ if(!files || count == 0)
+ return NULL;
+
+ int num_threads;
+ if(threads > count)
+ {
+ num_threads = count;
+ }
+ else if(threads > 0)
+ {
+ num_threads = threads;
+ }
+ else
+ {
+ num_threads = ph_num_threads();
+ }
+
+ DP **hashes = (DP**)malloc(count*sizeof(DP*));
+
+ for(int i = 0; i < count; ++i)
+ {
+ hashes[i] = (DP *)malloc(sizeof(DP));
+ hashes[i]->id = strdup(files[i]);
+ }
+
+ pthread_t thds[num_threads];
+
+ int rem = count % num_threads;
+ int start = 0;
+ int off = 0;
+ slice *s = new slice[num_threads];
+ for(int n = 0; n < num_threads; ++n)
+ {
+ off = (int)floor((count/(float)num_threads) + (rem>0?num_threads-(count % num_threads):0));
+
+ s[n].hash_p = &hashes[start];
+ s[n].n = off;
+ s[n].hash_params = new pair<int,int>(sr,channels);
+ start = off;
+ --rem;
+ pthread_create(&thds[n], NULL, ph_audio_thread, &s[n]);
+ }
+ for(int i = 0; i < num_threads; ++i)
+ {
+ pthread_join(thds[i], NULL);
+ delete (pair<int,int>*)s[i].hash_params;
+ }
+ delete[] s;
+
+ return hashes;
+
+}
+#endif
diff --git a/src/audiophash.h b/src/audiophash.h
index 2b41656..bbf5bf9 100644
--- a/src/audiophash.h
+++ b/src/audiophash.h
@@ -18,7 +18,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
*/
@@ -30,11 +30,12 @@
#include <unistd.h>
#include <stdlib.h>
#include <algorithm>
+#include "pHash.h"
extern "C" {
- #include "libavformat/avformat.h"
- #include "libavcodec/avcodec.h"
- #include "libswscale/swscale.h"
+ #include "./libavformat/avformat.h"
+ #include "./libavcodec/avcodec.h"
+ #include "./libswscale/swscale.h"
#include "ph_fft.h"
}
@@ -55,10 +56,12 @@ int ph_count_samples(const char *filename, int sr,int channels);
* /param filename - path and name of audio file to read
* /param sr - sample rate conversion
* /param channels - number channel conversion
- * /param N - (out) param for buf length
- * /param float* - float pointer to start of buffer, NULL if error
+ * /param buf - preallocated buffer
+ * /param buflen - (in/out) param for buf length
+ * /param nbsecs - float value for duration (in secs) to read from file
+ * /return float* - float pointer to start of buffer, NULL if error
*/
-float* ph_readaudio(const char *filename, int sr, int channels, int &N);
+float* ph_readaudio(const char *filename, int sr, int channels, float *sigbuf, int &buflen, const float nbsecs = 0);
@@ -74,9 +77,9 @@ float* ph_readaudio(const char *filename, int sr, int channels, int &N);
* /param nb_frames - (out) number of frames in audio buf and length of audiohash buffer returned
* /return uint32 pointer to audio hash, NULL for error
*/
-uint32_t* ph_audiohash(float *buf, int N, int sr, int &nb_frames);
-
+uint32_t* ph_audiohash(float *buf, int nbbuf, const int sr, int &nbframes);
+DP **ph_audio_hashes(char *files[], int count, int sr = 8000, int channels = 1, int threads = 0);
/* /brief bit count set bits in 32bit variable
* /param n
@@ -107,5 +110,4 @@ double ph_compare_blocks(const uint32_t *ptr_blockA,const uint32_t *ptr_blockB,
*/
double* ph_audio_distance_ber(uint32_t *hash_a , const int Na, uint32_t *hash_b, const int Nb, const float threshold, const int block_size, int &Nc);
-
#endif
diff --git a/src/cimgffmpeg.cpp b/src/cimgffmpeg.cpp
index cee72a8..fa7843c 100644
--- a/src/cimgffmpeg.cpp
+++ b/src/cimgffmpeg.cpp
@@ -18,7 +18,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
*/
@@ -382,7 +382,7 @@ long GetNumberVideoFrames(const char *file)
return nb_frames;
}
else { // frames must be counted
- AVPacket packet;
+ AVPacket packet;
nb_frames = (long)av_index_search_timestamp(str,str->duration, AVSEEK_FLAG_ANY|AVSEEK_FLAG_BACKWARD);
// Close the video file
av_close_input_file(pFormatCtx);
diff --git a/src/cimgffmpeg.h b/src/cimgffmpeg.h
index 8bea87b..d6f7fdd 100755
--- a/src/cimgffmpeg.h
+++ b/src/cimgffmpeg.h
@@ -18,7 +18,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
*/
@@ -33,9 +33,9 @@
#include "CImg.h"
extern "C" {
- #include "libavformat/avformat.h"
- #include "libavcodec/avcodec.h"
- #include "libswscale/swscale.h"
+ #include "./libavformat/avformat.h"
+ #include "./libavcodec/avcodec.h"
+ #include "./libswscale/swscale.h"
}
using namespace cimg_library;
diff --git a/src/pHash.cpp b/src/pHash.cpp
index 6c95f63..fbbb494 100644
--- a/src/pHash.cpp
+++ b/src/pHash.cpp
@@ -18,7 +18,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
*/
@@ -28,6 +28,39 @@
#include "cimgffmpeg.h"
#endif
+#ifdef HAVE_PTHREAD
+#include <pthread.h>
+
+int ph_num_threads()
+{
+ int numCPU = 1;
+ #ifdef linux
+ numCPU = sysconf( _SC_NPROCESSORS_ONLN );
+ #else
+ nt mib[4];
+ size_t len;
+
+ mib[0] = CTL_HW;
+ mib[1] = HW_AVAILCPU;
+
+ sysctl(mib, 2, &numCPU, &len, NULL, 0);
+
+ if( numCPU < 1 )
+ {
+ mib[1] = HW_NCPU;
+ sysctl( mib, 2, &numCPU, &len, NULL, 0 );
+
+ if( numCPU < 1 )
+ {
+ numCPU = 1;
+ }
+ }
+
+ #endif
+ return numCPU;
+}
+#endif
+
const char phash_project[] = "%s. Copyright 2008-2010 Aetilius, Inc.";
char phash_version[255] = {0};
const char* ph_about(){
@@ -222,7 +255,7 @@ int ph_crosscorr(const Digest &x,const Digest &y,double &pcc,double threshold){
#undef max
#endif
-int ph_image_digest(const CImg<uint8_t> &img,double sigma, double gamma,Digest &digest, int N){
+int _ph_image_digest(const CImg<uint8_t> &img,double sigma, double gamma,Digest &digest, int N){
int result = EXIT_FAILURE;
CImg<uint8_t> graysc;
@@ -270,7 +303,7 @@ int ph_image_digest(const char *file, double sigma, double gamma, Digest &digest
int res = -1;
if(src)
{
- int result = ph_image_digest(*src,sigma,gamma,digest,N);
+ int result = _ph_image_digest(*src,sigma,gamma,digest,N);
delete src;
res = result;
}
@@ -281,11 +314,11 @@ int _ph_compare_images(const CImg<uint8_t> &imA,const CImg<uint8_t> &imB,double
int result = 0;
Digest digestA;
- if (ph_image_digest(imA,sigma,gamma,digestA,N) < 0)
+ if (_ph_image_digest(imA,sigma,gamma,digestA,N) < 0)
goto cleanup;
Digest digestB;
- if (ph_image_digest(imB,sigma,gamma,digestB,N) < 0)
+ if (_ph_image_digest(imB,sigma,gamma,digestB,N) < 0)
goto cleanup;
if (ph_crosscorr(digestA,digestB,pcc,threshold) < 0)
@@ -296,8 +329,6 @@ int _ph_compare_images(const CImg<uint8_t> &imA,const CImg<uint8_t> &imB,double
cleanup:
- delete &imA;
- delete &imB;
free(digestA.coeffs);
free(digestB.coeffs);
return result;
@@ -310,6 +341,8 @@ int ph_compare_images(const char *file1, const char *file2,double &pcc, double s
int res = _ph_compare_images(*imA,*imB,pcc,sigma,gamma,N,threshold);
+ delete imA;
+ delete imB;
return res;
}
@@ -337,10 +370,15 @@ int ph_dct_imagehash(const char* file,ulong64 &hash){
}
CImg<float> meanfilter(7,7,1,1,1);
CImg<float> img;
- if (src.spectrum() >= 3){
+ if (src.spectrum() == 3){
img = src.RGBtoYCbCr().channel(0).get_convolve(meanfilter);
- } else if (img.spectrum() ==1){
- img = src.get_convolve(meanfilter);
+ } else if (src.spectrum() == 4){
+ int width = img.width();
+ int height = img.height();
+ int depth = img.depth();
+ img = src.crop(0,0,0,0,width-1,height-1,depth-1,2).RGBtoYCbCr().channel(0).get_convolve(meanfilter);
+ } else {
+ img = src.channel(0).get_convolve(meanfilter);
}
img.resize(32,32);
@@ -365,6 +403,78 @@ int ph_dct_imagehash(const char* file,ulong64 &hash){
return 0;
}
+
+#ifdef HAVE_PTHREAD
+void *ph_image_thread(void *p)
+{
+ slice *s = (slice *)p;
+ for(int i = 0; i < s->n; ++i)
+ {
+ DP *dp = (DP *)s->hash_p[i];
+ ulong64 hash;
+ int ret = ph_dct_imagehash(dp->id, hash);
+ dp->hash = (ulong64*)malloc(sizeof(hash));
+ memcpy(dp->hash, &hash, sizeof(hash));
+ dp->hash_length = 1;
+ }
+}
+
+DP** ph_dct_image_hashes(char *files[], int count, int threads)
+{
+ if(!files || count <= 0)
+ return NULL;
+
+ int num_threads;
+ if(threads > count)
+ {
+ num_threads = count;
+ }
+ else if(threads > 0)
+ {
+ num_threads = threads;
+ }
+ else
+ {
+ num_threads = ph_num_threads();
+ }
+
+ DP **hashes = (DP**)malloc(count*sizeof(DP*));
+
+ for(int i = 0; i < count; ++i)
+ {
+ hashes[i] = (DP *)malloc(sizeof(DP));
+ hashes[i]->id = strdup(files[i]);
+ }
+
+ pthread_t thds[num_threads];
+
+ int rem = count % num_threads;
+ int start = 0;
+ int off = 0;
+ slice *s = new slice[num_threads];
+ for(int n = 0; n < num_threads; ++n)
+ {
+ off = (int)floor((count/(float)num_threads) + (rem>0?num_threads-(count % num_threads):0));
+
+ s[n].hash_p = &hashes[start];
+ s[n].n = off;
+ s[n].hash_params = NULL;
+ start = off;
+ --rem;
+ pthread_create(&thds[n], NULL, ph_image_thread, &s[n]);
+ }
+ for(int i = 0; i < num_threads; ++i)
+ {
+ pthread_join(thds[i], NULL);
+ }
+ delete[] s;
+
+ return hashes;
+
+}
+#endif
+
+
#endif
#if defined(HAVE_VIDEO_HASH) && defined(HAVE_IMAGE_HASH)
@@ -584,6 +694,82 @@ ulong64* ph_dct_videohash(const char *filename, int &Length){
return hash;
}
+#ifdef HAVE_PTHREAD
+void *ph_video_thread(void *p)
+{
+ slice *s = (slice *)p;
+ for(int i = 0; i < s->n; ++i)
+ {
+ DP *dp = (DP *)s->hash_p[i];
+ int N;
+ ulong64 *hash = ph_dct_videohash(dp->id, N);
+ if(hash)
+ {
+ dp->hash = hash;
+ dp->hash_length = N;
+ }
+ else
+ {
+ dp->hash = NULL;
+ dp->hash_length = 0;
+ }
+ }
+}
+
+DP** ph_dct_video_hashes(char *files[], int count, int threads)
+{
+ if(!files || count <= 0)
+ return NULL;
+
+ int num_threads;
+ if(threads > count)
+ {
+ num_threads = count;
+ }
+ else if(threads > 0)
+ {
+ num_threads = threads;
+ }
+ else
+ {
+ num_threads = ph_num_threads();
+ }
+
+ DP **hashes = (DP**)malloc(count*sizeof(DP*));
+
+ for(int i = 0; i < count; ++i)
+ {
+ hashes[i] = (DP *)malloc(sizeof(DP));
+ hashes[i]->id = strdup(files[i]);
+ }
+
+ pthread_t thds[num_threads];
+
+ int rem = count % num_threads;
+ int start = 0;
+ int off = 0;
+ slice *s = new slice[num_threads];
+ for(int n = 0; n < num_threads; ++n)
+ {
+ off = (int)floor((count/(float)num_threads) + (rem>0?num_threads-(count % num_threads):0));
+
+ s[n].hash_p = &hashes[start];
+ s[n].n = off;
+ s[n].hash_params = NULL;
+ start = off;
+ --rem;
+ pthread_create(&thds[n], NULL, ph_video_thread, &s[n]);
+ }
+ for(int i = 0; i < num_threads; ++i)
+ {
+ pthread_join(thds[i], NULL);
+ }
+ delete[] s;
+
+ return hashes;
+
+}
+#endif
double ph_dct_videohash_dist(ulong64 *hashA, int N1, ulong64 *hashB, int N2, int threshold){
@@ -626,26 +812,19 @@ int ph_hamming_distance(const ulong64 hash1,const ulong64 hash2){
return (x * h01)>>56;
}
-DP* ph_malloc_datapoint(int hashtype, int pathlength){
+DP* ph_malloc_datapoint(int hashtype){
DP* dp = (DP*)malloc(sizeof(DP));
dp->hash = NULL;
dp->id = NULL;
- dp->path = (float*)calloc(pathlength,sizeof(float));
dp->hash_type = hashtype;
-
return dp;
}
void ph_free_datapoint(DP *dp){
- if (!dp)
+ if (!dp){
return;
- if (dp->path)
- free(dp->path);
- if (dp->id)
- free(dp->id);
- if (dp->hash)
- free(dp->hash);
+ }
free(dp);
-
+ dp=NULL;
return;
}
@@ -686,7 +865,7 @@ DP** ph_read_imagehashes(const char *dirname,int pathlength, int &count){
strcat(path, dir_entry->d_name);
if (ph_dct_imagehash(path, tmphash) < 0) //calculate the hash
continue;
- dp = ph_malloc_datapoint(UINT64ARRAY,pathlength);
+ dp = ph_malloc_datapoint(UINT64ARRAY);
dp->id = strdup(path);
dp->hash = (void*)&tmphash;
dp->hash_length = 1;
@@ -739,8 +918,22 @@ char** ph_readfilenames(const char *dirname,int &count){
return files;
}
-
-uint8_t* ph_mh_imagehash(const char *filename, int &N,int alpha, int lvl){
+CImg<float>* GetMHKernel(float alpha, float level){
+ int sigma = (int)4*pow((float)alpha,(float)level);
+ static CImg<float> *pkernel = NULL;
+ float xpos, ypos, A;
+ if (!pkernel){
+ pkernel = new CImg<float>(2*sigma+1,2*sigma+1,1,1,0);
+ cimg_forXY(*pkernel,X,Y){
+ xpos = pow(alpha,-level)*(X-sigma);
+ ypos = pow(alpha,-level)*(Y-sigma);
+ A = xpos*xpos + ypos*ypos;
+ pkernel->atXY(X,Y) = (2-A)*exp(-A/2);
+ }
+ }
+ return pkernel;
+}
+uint8_t* ph_mh_imagehash(const char *filename, int &N,float alpha, float lvl){
if (filename == NULL){
return NULL;
}
@@ -749,58 +942,50 @@ uint8_t* ph_mh_imagehash(const char *filename, int &N,int alpha, int lvl){
CImg<uint8_t> src(filename);
CImg<uint8_t> img;
+
if (img.spectrum() == 3){
- img = src.get_RGBtoYCbCr().channel(0).blur(1.5,1.5,1.5).resize(512,512,1,1,5).get_equalize(256);
+ img = src.get_RGBtoYCbCr().channel(0).blur(1.0).resize(512,512,1,1,5).get_equalize(256);
} else{
- img = src.get_blur(1.5,1.5,1.5).resize(512,512,1,1,5);
+ img = src.channel(0).get_blur(1.0).resize(512,512,1,1,5).get_equalize(256);
}
src.clear();
- int sigma = (int)(4*pow((float)alpha,(float)lvl));
- float xpos, ypos, A;
- CImg<float> MHKernel(2*sigma+1,2*sigma+1,1,1,0);
- cimg_forXY(MHKernel,X,Y){
- xpos = pow((float)alpha,-lvl)*(X - sigma);
- ypos = pow((float)alpha,-lvl)*(Y - sigma);
- A = xpos*xpos + ypos*ypos;
- MHKernel(X,Y) = (2-A)*exp(-A/2);
- }
-
- CImg<float> fresp = img.get_correlate(MHKernel);
+
+ CImg<float> *pkernel = GetMHKernel(alpha,lvl);
+ CImg<float> fresp = img.get_correlate(*pkernel);
img.clear();
fresp.normalize(0,1.0);
CImg<float> blocks(31,31,1,1,0);
for (int rindex=0;rindex < 31;rindex++){
- for (int cindex=0;cindex < 31;cindex++){
- blocks(rindex,cindex) = fresp.get_crop(rindex*16,cindex*16,rindex*16+16-1,cindex*16+16-1).sum();
- }
+ for (int cindex=0;cindex < 31;cindex++){
+ blocks(rindex,cindex) = fresp.get_crop(rindex*16,cindex*16,rindex*16+16-1,cindex*16+16-1).sum();
+ }
}
-
int hash_index;
int nb_ones = 0, nb_zeros = 0;
int bit_index = 0;
unsigned char hashbyte = 0;
for (int rindex=0;rindex < 31-2;rindex+=4){
- CImg<float> subsec;
- for (int cindex=0;cindex < 31-2;cindex+=4){
- subsec = blocks.get_crop(cindex,rindex, cindex+2, rindex+2).unroll('x');
- float ave = subsec.mean();
- cimg_forX(subsec, I){
- hashbyte <<= 1;
- if (subsec(I) > ave){
- hashbyte |= 0x01;
- nb_ones++;
- } else {
- nb_zeros++;
- }
- bit_index++;
- if ((bit_index%8) == 0){
- hash_index = (int)(bit_index/8) - 1;
- hash[hash_index] = hashbyte;
- hashbyte = 0x00;
+ CImg<float> subsec;
+ for (int cindex=0;cindex < 31-2;cindex+=4){
+ subsec = blocks.get_crop(cindex,rindex, cindex+2, rindex+2).unroll('x');
+ float ave = subsec.mean();
+ cimg_forX(subsec, I){
+ hashbyte <<= 1;
+ if (subsec(I) > ave){
+ hashbyte |= 0x01;
+ nb_ones++;
+ } else {
+ nb_zeros++;
+ }
+ bit_index++;
+ if ((bit_index%8) == 0){
+ hash_index = (int)(bit_index/8) - 1;
+ hash[hash_index] = hashbyte;
+ hashbyte = 0x00;
+ }
+ }
}
- }
}
- }
return hash;
}
@@ -835,25 +1020,31 @@ double ph_hammingdistance2(uint8_t *hashA, int lenA, uint8_t *hashB, int lenB){
}
-int ph_selectvantagepoints(MVPFile *m, DP **points, int N, int &sv1_pos, int &sv2_pos, float &maxdist, float &mindist){
- sv1_pos = 0;
- sv2_pos = 0;
- maxdist = 0.0;
- mindist = INT_MAX;
- float d;
- for (int i=0;i<N;i++){ /* find 2 points furthest apart */
- for (int j=i+1;j<N;j++){
- d = m->hashdist(points[i],points[j]);
- if (d > maxdist){
- maxdist = d;
- sv1_pos = i;
- sv2_pos = j;
-
- }
- if (m->hashdist(points[i],points[j]) < mindist){
- mindist = d;
+int ph_selectvantagepoints(MVPFile *m, DP **points, int N, int &sv1_pos, int &sv2_pos){
+ if (N > 2){
+ sv1_pos = 0;
+ sv2_pos = 1;
+ float maxdist = 0.0f;
+ float d;
+ for (int i=0;i<N;i++){ /* find 2 points furthest apart */
+ for (int j=i+1;j<N;j++){
+ d = m->hashdist(points[i],points[j]);
+ if (d > maxdist){
+ maxdist = d;
+ sv1_pos = i;
+ sv2_pos = j;
+ }
}
}
+ } else if (N > 1){
+ sv1_pos = 0;
+ sv2_pos = 1;
+ } else if (N > 0){
+ sv1_pos = 0;
+ sv2_pos = -1;
+ } else {
+ sv1_pos = -1;
+ sv2_pos = -1;
}
return 0;
}
@@ -863,49 +1054,62 @@ DP* ph_read_datapoint(MVPFile *m){
uint8_t active;
uint16_t byte_len;
uint16_t id_len;
- uint16_t hash_len;
- int type = m->hash_type;
- int PathLength = m->pathlength;
- off_t file_pos = m->file_pos;
+ uint32_t hash_len;
off_t offset_mask = m->pgsize - 1;
- memcpy(&active, &(m->buf[file_pos & offset_mask]), sizeof(uint8_t));
- file_pos++;
+ memcpy(&active, &(m->buf[m->file_pos & offset_mask]), sizeof(uint8_t));
+ m->file_pos++;
- memcpy(&byte_len,&(m->buf[file_pos & offset_mask]), sizeof(uint16_t));
- file_pos += sizeof(uint16_t);
+ memcpy(&byte_len,&(m->buf[m->file_pos & offset_mask]), sizeof(uint16_t));
+ m->file_pos += sizeof(uint16_t);
if ((active == 0) ||(byte_len == 0)){
return dp;
}
- dp = ph_malloc_datapoint(type,PathLength);
+ dp = ph_malloc_datapoint(m->hash_type);
+ dp->path = (float*)malloc((m->pathlength)*sizeof(float));
- memcpy(&id_len,&(m->buf[file_pos & offset_mask]), sizeof(uint16_t));
- file_pos += sizeof(uint16_t);
+ memcpy(&id_len,&(m->buf[m->file_pos & offset_mask]), sizeof(uint16_t));
+ m->file_pos += sizeof(uint16_t);
dp->id = (char*)malloc((id_len+1)*sizeof(uint8_t));
- memcpy(dp->id, &(m->buf[file_pos & offset_mask]), id_len);
+ memcpy(dp->id, &(m->buf[m->file_pos & offset_mask]), id_len);
dp->id[id_len] = '\0';
- file_pos += id_len;
+ m->file_pos += id_len;
- memcpy(&hash_len, &(m->buf[file_pos & offset_mask]), sizeof(uint16_t));
+ memcpy(&hash_len, &(m->buf[m->file_pos & offset_mask]), sizeof(uint32_t));
dp->hash_length = hash_len;
- file_pos += sizeof(uint16_t);
+ m->file_pos += sizeof(uint32_t);
- dp->hash = malloc(hash_len*type);
- memcpy(dp->hash, &(m->buf[file_pos & offset_mask]), hash_len*type);
- file_pos += hash_len*type;
+ dp->hash = malloc(hash_len*(m->hash_type));
+ memcpy(dp->hash, &(m->buf[m->file_pos & offset_mask]), hash_len*(m->hash_type));
+ m->file_pos += hash_len*(m->hash_type);
- memcpy(dp->path, &(m->buf[file_pos & offset_mask]), PathLength*sizeof(float));
- file_pos += PathLength*sizeof(float);
-
- m->file_pos = file_pos;
+ memcpy(dp->path, &(m->buf[m->file_pos & offset_mask]), (m->pathlength)*sizeof(float));
+ m->file_pos += (m->pathlength)*sizeof(float);
return dp;
}
+void ph_mvp_init(MVPFile *m)
+{
+ if(!m) return;
+
+ m->branchfactor = 2;
+ m->pathlength = 5;
+ m->leafcapacity = 23;
+ #ifdef linux
+ m->pgsize = sysconf(_SC_PAGE_SIZE);
+ #else
+ m->pgsize = getpagesize();
+ #endif
+
+}
int ph_sizeof_dp(DP *dp,MVPFile *m){
- return (strlen(dp->id) + 4 + (dp->hash_length)*(m->hash_type) + (m->pathlength)*sizeof(float));
+ if (dp == NULL){
+ return 3; /* for byte_len(uint16_t) and active (uint8_t) flags */
+ }
+ return (strlen(dp->id) + 9 + (dp->hash_length)*(m->hash_type) + (m->pathlength)*sizeof(float));
}
off_t ph_save_datapoint(DP *dp, MVPFile *m){
@@ -924,8 +1128,8 @@ off_t ph_save_datapoint(DP *dp, MVPFile *m){
int type = m->hash_type;
int PathLength = m->pathlength;
uint16_t id_len = strlen(dp->id);
- uint16_t hash_len = dp->hash_length;
- byte_len = id_len + 4 + hash_len*(m->hash_type) + PathLength*sizeof(float);
+ uint32_t hash_len = dp->hash_length;
+ byte_len = id_len + 6 + hash_len*(m->hash_type) + PathLength*sizeof(float);
memcpy(&(m->buf[m->file_pos & offset_mask]), &active, 1);
m->file_pos++;
@@ -939,8 +1143,8 @@ off_t ph_save_datapoint(DP *dp, MVPFile *m){
memcpy(&(m->buf[m->file_pos & offset_mask]), (dp->id), id_len);
m->file_pos += id_len;
- memcpy(&(m->buf[m->file_pos & offset_mask]), &hash_len, sizeof(uint16_t));
- m->file_pos += sizeof(uint16_t);
+ memcpy(&(m->buf[m->file_pos & offset_mask]), &hash_len, sizeof(uint32_t));
+ m->file_pos += sizeof(uint32_t);
memcpy(&(m->buf[m->file_pos & offset_mask]), (dp->hash), hash_len*type);
m->file_pos += hash_len*type;
@@ -954,70 +1158,53 @@ off_t ph_save_datapoint(DP *dp, MVPFile *m){
}
-MVPFile* _ph_map_mvpfile(uint8_t filenumber, off_t offset, MVPFile *m){
- MVPFile *ret_file = NULL;
+MVPRetCode _ph_map_mvpfile(uint8_t filenumber, off_t offset, MVPFile *m,MVPFile *m2){
- if (filenumber == 0){ /* in the file denoted by m , must advance to page containing offset */
+ m2->filename = strdup(m->filename);
+ m2->branchfactor = m->branchfactor;
+ m2->leafcapacity = m->leafcapacity;
+ m2->pathlength = m->pathlength;
+ m2->hash_type = m->hash_type;
+ m2->hashdist = m->hashdist;
+ m2->pgsize = m->pgsize;
+ m2->nbdbfiles = m->nbdbfiles;
+ m2->file_pos = offset;
+ m2->filenumber = filenumber;
+ if (filenumber == m->filenumber){ /* advance to offset in same file */
off_t page_mask = ~(m->pgsize - 1);
off_t page_offset = offset & page_mask;
- ret_file = m;
- m->file_pos = offset;
- munmap(m->buf, m->pgsize);
- m->buf = (char*)mmap(NULL,m->pgsize,PROT_WRITE|PROT_READ,MAP_SHARED,m->fd, page_offset);
- madvise(m->buf,m->pgsize,MADV_SEQUENTIAL);
- if (m->buf == MAP_FAILED){
- return NULL;
+ m2->fd = m->fd;
+ m2->buf = (char*)mmap(NULL,m2->pgsize,PROT_WRITE|PROT_READ,MAP_SHARED,m2->fd, page_offset);
+ madvise(m2->buf,m2->pgsize,MADV_SEQUENTIAL);
+ if (m2->buf == MAP_FAILED){
+ return PH_ERRMMAP;
}
} else { /* open and map to new file denoted by m->filename and filenumber */
off_t page_mask = ~(m->pgsize - 1);
- ret_file = (MVPFile*)malloc(sizeof(MVPFile));
- if (!ret_file)
- return NULL;
char extfile[256];
snprintf(extfile, sizeof(extfile),"%s%d.mvp", m->filename, filenumber);
- ret_file->filename = strdup(m->filename);
- ret_file->fd = open(extfile, O_RDWR);
- ret_file->pathlength = m->pathlength;
- ret_file->hash_type = m->hash_type;
- ret_file->hashdist = m->hashdist;
- ret_file->branchfactor = m->branchfactor;
- ret_file->leafcapacity = m->leafcapacity;
- ret_file->nbdbfiles = m->nbdbfiles;
- ret_file->pgsize = m->pgsize;
- ret_file->file_pos = offset;
+ m2->fd = open(extfile, O_RDWR);
off_t page_offset = offset & page_mask;
-
- ret_file->buf = (char*)mmap(NULL, ret_file->pgsize, PROT_READ|PROT_WRITE, MAP_SHARED,
- ret_file->fd, page_offset);
- if (ret_file->buf == MAP_FAILED){
- free(ret_file);
- return NULL;
+ m2->buf = (char*)mmap(NULL, m2->pgsize, PROT_READ|PROT_WRITE, MAP_SHARED, m2->fd, page_offset);
+ if (m2->buf == MAP_FAILED){
+ return PH_ERRMMAP;
}
- madvise(ret_file->buf, ret_file->pgsize, MADV_SEQUENTIAL);
+ madvise(m2->buf, m2->pgsize, MADV_SEQUENTIAL);
}
- return ret_file;
+ return PH_SUCCESS;
}
-void _ph_unmap_mvpfile(uint8_t filenumber, off_t orig_pos, MVPFile *m, MVPFile *m2){
-
- if (filenumber == 0){ /*remap to same main file */
- off_t page_mask = ~(m->pgsize - 1);
- munmap(m->buf, m->pgsize);
- m->file_pos = orig_pos;
- off_t pg_offset = orig_pos & page_mask;
- m->buf = (char*)mmap(NULL, m->pgsize, PROT_WRITE|PROT_READ, MAP_SHARED,m->fd,pg_offset);
- if (m->buf == MAP_FAILED){
- return;
- }
- madvise(m->buf,m->pgsize,MADV_SEQUENTIAL);
- } else { /*remap to back to main file */
- munmap(m2->buf, m2->pgsize);
+MVPRetCode _ph_unmap_mvpfile(uint8_t filenumber, off_t orig_pos, MVPFile *m, MVPFile *m2){
+ free(m2->filename);
+ msync(m2->buf,m2->pgsize,MS_SYNC);
+ munmap(m2->buf,m2->pgsize);
+ if (m->filenumber != m2->filenumber){
close(m2->fd);
- free(m2);
}
+ return PH_SUCCESS;
}
@@ -1038,21 +1225,18 @@ float hammingdistance(DP *pntA, DP *pntB){
}
-MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
- DP **results, int *count, int level){
- int BranchFactor = m->branchfactor;
- int LengthM1 = BranchFactor-1;
- int LengthM2 = BranchFactor*LengthM1;
- int PathLength = m->pathlength;
+MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius, float threshold,
+ DP **results, int &nbfound, int level){
MVPRetCode ret = PH_SUCCESS;
-
- if ( (!m)||(!query)||(!results)||(!count)||(level < 0) )
- return PH_ERRARG;
+ int LengthM1 = m->branchfactor - 1;
+ int LengthM2 = (m->branchfactor)*(m->branchfactor-1);
+ if ( (!m)||(!query)||(!results)||(level < 0) )
+ return PH_ERRNULLARG;
hash_compareCB hashdist = m->hashdist;
if (!hashdist)
- return PH_ERRARG;
+ return PH_ERRNULLARG;
off_t offset_mask, page_mask;
@@ -1067,31 +1251,44 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
DP *sv1 = ph_read_datapoint(m);
DP *sv2 = ph_read_datapoint(m);
float d1 = hashdist(query,sv1);
-
/* check if distance(sv1,query) <= radius */
- if (d1 <= radius){
- results[(*count)++] = sv1;
- if (*count >= knearest)
+ if (d1 <= threshold){
+ results[nbfound++] = sv1;
+ if ((nbfound) >= knearest){
return PH_ERRCAP;
+ }
} else {
+ free(sv1->id);
+ free(sv1->path);
+ free(sv1->hash);
ph_free_datapoint(sv1);
}
-
+
if (sv2){
float d2 = hashdist(query,sv2);
/* check if distance(sv2,query) <= radius */
- if (d2 <= radius){
- results[(*count)++] = sv2;
- if (*count >= knearest)
+ if (d2 <= threshold){
+ results[nbfound++] = sv2;
+ if (nbfound >= knearest){
return PH_ERRCAP;
+ }
} else {
+ free(sv2->id);
+ free(sv2->path);
+ free(sv2->hash);
ph_free_datapoint(sv2);
}
-
+ if (level < m->pathlength){
+ query->path[level] = d1;
+ }
+ if (level+1 < m->pathlength){
+ query->path[level+1] = d2;
+ }
+
uint8_t Np;
memcpy(&Np, &m->buf[m->file_pos & offset_mask], sizeof(uint8_t));
m->file_pos += sizeof(uint8_t);
- off_t curr_pos;
+ off_t curr_pos;
/* read in each datapoint in the leaf - only retrieve the point if it
dist(sv1,dp)=da and dist(sv2,dp)=db cannot preclude the point */
@@ -1113,26 +1310,29 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
/* test each path[] distance and as soon as one does not fit
disclude the point */
+
if (dp){
- for (int j=0;j<level;j++){
- if ((query->path[j]-radius <= dp->path[j])
- &&(query->path[j]+radius >= dp->path[j])){
- if (hashdist(query,dp) > radius){
+ int pl = (level < m->pathlength) ? level : m->pathlength;
+ for (int j=0;j<pl;j++){
+ if (!((query->path[j]-radius <= dp->path[j])
+ &&(query->path[j]+radius >= dp->path[j]))){
include = 0;
break;
- }
}
}
- } else {
- include = 0;
- }
- if (include){
- results[(*count)++] = dp;
- if (*count >= knearest)
+ }
+ if (include && (hashdist(query,dp) <= threshold)){
+ results[nbfound++] = dp;
+ if (nbfound >= knearest){
return PH_ERRCAP;
- } else {
+ }
+ } else {
+ free(dp->id);
+ free(dp->path);
+ free(dp->hash);
ph_free_datapoint(dp);
}
+
} else {
m->file_pos += sizeof(off_t);
}
@@ -1142,14 +1342,15 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
/* read sv1, sv2 */
DP *sv1 = ph_read_datapoint(m);
DP *sv2 = ph_read_datapoint(m);
+
/* read 1st and 2nd level pivots */
float *M1 = (float*)malloc(LengthM1*sizeof(float));
if (!M1){
- return PH_ERRMEM;
+ return PH_ERRMEMALLOC;
}
float *M2 = (float*)malloc(LengthM2*sizeof(float));
if (!M2){
- return PH_ERRMEM;
+ return PH_ERRMEMALLOC;
}
memcpy(M1, &m->buf[m->file_pos & offset_mask], LengthM1*sizeof(float));
@@ -1159,27 +1360,35 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
float d1 = hashdist(query, sv1);
float d2 = hashdist(query, sv2);
-
+
/* fill in path values in query */
- if (level < PathLength)
+ if (level < m->pathlength)
query->path[level] = d1;
- if (level < PathLength - 1)
+ if (level+1 < m->pathlength)
query->path[level+1] = d2;
/* check if sv1 sv2 are close enough to query */
- if (d1 <= radius){
- results[(*count)++] = sv1;
- if (*count >= knearest)
+ if (d1 <= threshold){
+ results[nbfound++] = sv1;
+ if (nbfound >= knearest){
return PH_ERRCAP;
+ }
} else {
+ free(sv1->id);
+ free(sv1->hash);
+ free(sv1->path);
ph_free_datapoint(sv1);
}
- if (d2 <= radius){
- results[(*count)++] = sv2;
- if (*count >= knearest)
+ if (d2 <= threshold){
+ results[nbfound++] = sv2;
+ if (nbfound >= knearest){
return PH_ERRCAP;
+ }
} else {
+ free(sv2->id);
+ free(sv2->hash);
+ free(sv2->path);
ph_free_datapoint(sv2);
}
@@ -1190,6 +1399,7 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
off_t child_pos, curr_pos;
off_t start_pos = m->file_pos;
off_t orig_pos;
+ MVPFile m2;
/* check <= each M1 pivot */
for (pivot1=0;pivot1 < LengthM1;pivot1++){
if (d1-radius <= M1[pivot1]){
@@ -1197,22 +1407,25 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
for (pivot2=0;pivot2<LengthM1;pivot2++){
if (d2 - radius <= M2[pivot2+pivot1*LengthM1]){
/*determine pos from which to read filenumber and offset */
- curr_pos = start_pos + (pivot2+pivot1*BranchFactor)*(sizeof(uint8_t)+sizeof(off_t));
+ curr_pos = start_pos + (pivot2+pivot1*(m->branchfactor))*(sizeof(uint8_t)+sizeof(off_t));
m->file_pos = curr_pos;
memcpy(&filenumber,&(m->buf[m->file_pos&offset_mask]),sizeof(uint8_t));
m->file_pos++;
memcpy(&child_pos,&(m->buf[m->file_pos&offset_mask]),sizeof(off_t));
m->file_pos += sizeof(off_t);
+ if ((filenumber == 0) && (child_pos == 0)){
+ continue;
+ }
/*save position and remap to new file/position */
- orig_pos = m->file_pos;
- MVPFile *m2 = _ph_map_mvpfile(filenumber,child_pos, m);
- if (m2){
- ret = _ph_query_mvptree(m2,query,knearest,radius,results,count,level+2);
+ ret = _ph_map_mvpfile(filenumber,child_pos, m, &m2);
+ if (ret == PH_SUCCESS){
+ ret = _ph_query_mvptree(&m2,query,knearest,radius,threshold,results,nbfound,level+2);
+ /* unmap and remap to the origional file/position */
+ ret = _ph_unmap_mvpfile(filenumber, orig_pos, m, &m2);
+ } else {
+ goto querycleanup;
}
-
- /* unmap and remap to the origional file/posion */
- _ph_unmap_mvpfile(filenumber, orig_pos, m, m2);
}
}
@@ -1220,22 +1433,25 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
if (d2+radius >= M2[LengthM1-1+pivot1*LengthM1]){
/*determine position from which to read filenumber and offset */
- curr_pos = start_pos + (BranchFactor-1+pivot1*BranchFactor)*(sizeof(uint8_t)+sizeof(off_t));
+ curr_pos = start_pos + (m->branchfactor-1+pivot1*(m->branchfactor))*(sizeof(uint8_t)+sizeof(off_t));
m->file_pos = curr_pos;
memcpy(&filenumber,&m->buf[m->file_pos&offset_mask],sizeof(uint8_t));
m->file_pos++;
memcpy(&child_pos,&m->buf[m->file_pos&offset_mask],sizeof(off_t));
m->file_pos += sizeof(off_t);
- /*saveposition and remap to new file/position */
- orig_pos = m->file_pos;
+ if ((filenumber == 0) && (child_pos == 0)){
+ continue;
+ }
- MVPFile *m2 = _ph_map_mvpfile(filenumber, child_pos,m);
- if (m2){
- ret = _ph_query_mvptree(m2,query,knearest,radius,results,count,level+2);
+ ret = _ph_map_mvpfile(filenumber, child_pos,m, &m2);
+ if (ret == PH_SUCCESS){
+ ret = _ph_query_mvptree(&m2,query,knearest,radius,threshold,results,nbfound,level+2);
+ /*unmap and remap to original file/position */
+ _ph_unmap_mvpfile(filenumber, orig_pos, m, &m2);
+ } else {
+ goto querycleanup;
}
- /*unmap and remap to original file/position */
- _ph_unmap_mvpfile(filenumber, orig_pos, m, m2);
}
}
}
@@ -1247,21 +1463,26 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
if (d2-radius <= M2[pivot2+LengthM1*LengthM1]){
/*determine pos from which to read filenumber and position */
- curr_pos = start_pos + (pivot2+LengthM1*BranchFactor)*(sizeof(uint8_t)+sizeof(off_t));
+ curr_pos = start_pos + (pivot2+LengthM1*(m->branchfactor))*(sizeof(uint8_t)+sizeof(off_t));
m->file_pos = curr_pos;
memcpy(&filenumber,&m->buf[m->file_pos&offset_mask],sizeof(uint8_t));
m->file_pos++;
memcpy(&child_pos,&m->buf[m->file_pos&offset_mask],sizeof(off_t));
m->file_pos += sizeof(off_t);
+
+ if ((filenumber == 0)&&(child_pos == 0)){
+ continue;
+ }
/*save file position and remap to new filenumber/offset */
- orig_pos = m->file_pos;
- MVPFile *m2 = _ph_map_mvpfile(filenumber, child_pos, m);
- if (m2){
- ret = _ph_query_mvptree(m2,query,knearest,radius,results,count,level+2);
+ ret = _ph_map_mvpfile(filenumber, child_pos, m, &m2);
+ if (ret == PH_SUCCESS){
+ ret = _ph_query_mvptree(&m2,query,knearest,radius,threshold,results,nbfound,level+2);
+ /* unmap/remap to original filenumber/position */
+ _ph_unmap_mvpfile(filenumber, orig_pos, m, &m2);
+ } else {
+ goto querycleanup;
}
- /* unmap/remap to original filenumber/position */
- _ph_unmap_mvpfile(filenumber, orig_pos, m, m2);
}
}
@@ -1269,7 +1490,7 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
if (d2+radius >= M2[LengthM1-1+LengthM1*LengthM1]){
/* determine position from which to read filenumber and child position */
- curr_pos = start_pos + (BranchFactor-1+LengthM1*BranchFactor)*(sizeof(uint8_t)+sizeof(off_t));
+ curr_pos = start_pos + (m->branchfactor-1+LengthM1*(m->branchfactor))*(sizeof(uint8_t)+sizeof(off_t));
m->file_pos = curr_pos;
memcpy(&filenumber,&m->buf[m->file_pos&offset_mask],sizeof(uint8_t));
m->file_pos += sizeof(uint8_t);
@@ -1277,24 +1498,34 @@ MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
m->file_pos += sizeof(off_t);
/* save position and remap to new filenumber/position */
- orig_pos = m->file_pos;
- MVPFile *m2 = _ph_map_mvpfile(filenumber, child_pos, m);
- if (m2){
- ret = _ph_query_mvptree(m2,query,knearest,radius,results,count,level+2);
+ if (!(filenumber == 0 && child_pos == 0)){
+ ret = _ph_map_mvpfile(filenumber, child_pos, m, &m2);
+ if (ret == PH_SUCCESS){
+ ret = _ph_query_mvptree(&m2,query,knearest,radius,threshold, results,nbfound,level+2);
+ /* return to original and remap to original filenumber/position */
+ ret = _ph_unmap_mvpfile(filenumber, orig_pos, m, &m2);
+ } else {
+ goto querycleanup;
+ }
+ } else {
+ goto querycleanup;
}
- /* return to original and remap to original filenumber/position */
- _ph_unmap_mvpfile(filenumber, orig_pos, m, m2);
}
}
+
+querycleanup:
+ free(M1);
+ free(M2);
} else { /* unrecognized node */
ret = PH_ERRNTYPE;
}
+
return ret;
}
-MVPRetCode ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
- DP **results, int *count){
+MVPRetCode ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius, float threshold,
+ DP **results, int &nbfound){
/*use host pg size until file pg size used can be determined */
m->pgsize = sysconf(_SC_PAGESIZE);
@@ -1302,8 +1533,9 @@ MVPRetCode ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
snprintf(mainfile, sizeof(mainfile),"%s.mvp", m->filename);
m->fd = open(mainfile, O_RDWR);
if (m->fd < 0){
- return PH_ERRFILE;
+ return PH_ERRFILEOPEN;
}
+
m->file_pos = 0;
m->buf=(char*)mmap(NULL,m->pgsize,PROT_READ|PROT_WRITE,MAP_SHARED,m->fd,m->file_pos);
if (m->buf == MAP_FAILED){
@@ -1339,6 +1571,17 @@ MVPRetCode ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
memcpy(&type, &m->buf[m->file_pos++], 1);
+ query->path = (float*)malloc(p*sizeof(float));
+ if (query->path == NULL){
+ return PH_ERRMEMALLOC;
+ }
+ m->branchfactor = bf;
+ m->pathlength = p;
+ m->leafcapacity = k;
+ m->hash_type = (HashType)type;
+ m->nbdbfiles = nbdbfiles;
+
+
#ifdef HAVE_MREMAP
m->buf = (char*)mremap(m->buf,m->pgsize,int_pgsize,MREMAP_MAYMOVE);
#else
@@ -1352,22 +1595,28 @@ MVPRetCode ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
m->pgsize = int_pgsize;
m->file_pos = HeaderSize;
-
+ m->filenumber = 0;
/* finish the query by calling the recursive auxiliary function */
- *count = 0;
- MVPRetCode res = _ph_query_mvptree(m,query,knearest,radius,results,count,0);
+ nbfound = 0;
+ MVPRetCode res = _ph_query_mvptree(m,query,knearest,radius,threshold, results,nbfound,0);
munmap(m->buf, m->pgsize);
m->buf = NULL;
close(m->fd);
+ free(query->path);
+
+ for (int i=0;i<nbfound;i++){
+ free(results[i]->path);
+ }
+
m->fd = 0;
m->file_pos = 0;
-
+
return res;
}
-FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_flag, int level){
+MVPRetCode _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_flag, int level, FileIndex *pOffset){
int Np = (nbpoints >= 2) ? nbpoints - 2 : 0;
int BranchFactor = m->branchfactor;
int PathLength = m->pathlength;
@@ -1375,19 +1624,17 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
int LengthM1 = BranchFactor-1;
int LengthM2 = BranchFactor*LengthM1;
int Fanout = BranchFactor*BranchFactor;
+ hash_compareCB hashdist = m->hashdist;
- if ((!m) || (!points) || (nbpoints <= 0))
- return (FileIndex*)PH_ERRARG;
- FileIndex *pOffset = (FileIndex*)malloc(sizeof(FileIndex));
- if (!pOffset){
- return (FileIndex*)PH_ERRMEM;
+ if ((!m) || (!points) || (nbpoints < 0) || (hashdist == NULL)){
+ return PH_ERRNULLARG;
}
- hash_compareCB hashdist = m->hashdist;
- if (hashdist == NULL){
- free(pOffset);
- return (FileIndex*)PH_ERRARG;
+ if (nbpoints == 0){
+ pOffset->fileno = 0;
+ pOffset->offset = 0;
+ return PH_SUCCESS;
}
-
+ MVPRetCode ret = PH_SUCCESS;
off_t offset_mask = m->pgsize - 1;
off_t page_mask = ~(m->pgsize - 1);
if (nbpoints <= LeafCapacity + 2){ /* leaf */
@@ -1405,8 +1652,7 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
m2.fd = open(extfile, O_CREAT|O_RDWR|O_APPEND, 00755);
if (m2.fd < 0){
- free(pOffset);
- return NULL;
+ return PH_ERRFILEOPEN;
}
m2.hash_type = m->hash_type;
@@ -1421,8 +1667,7 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
m2.fd = open(extfile,O_CREAT|O_RDWR|O_APPEND, 00755);
if (m2.fd < 0){
- free(pOffset);
- return NULL;
+ return PH_ERRFILEOPEN;
}
fstat(m2.fd, &fileinfo);
}
@@ -1430,12 +1675,10 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
m2.file_pos = lseek(m2.fd, 0, SEEK_END);
if (m2.file_pos < 0){
- free(pOffset);
- return NULL;
+ return PH_ERRFILESEEK;
}
if (ftruncate(m2.fd, m2.file_pos + m->pgsize) < 0){
- free(pOffset);
- return NULL;
+ return PH_ERRFILETRUNC;
}
off_t end_pos = m2.file_pos + m->pgsize;
@@ -1446,8 +1689,7 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
m2.buf = (char*)mmap(NULL,m->pgsize,PROT_READ|PROT_WRITE,MAP_SHARED,m2.fd,pa_offset);
if (m2.buf == MAP_FAILED){
close(m2.fd);
- free(pOffset);
- return NULL;
+ return PH_ERRMMAP;
}
m2.buf[m2.file_pos & offset_mask] = ntype;
@@ -1455,20 +1697,21 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
/* find vantage points, sv1 and sv2 */
int sv1_pos, sv2_pos;
- float max_distance, min_distance;
- ph_selectvantagepoints(&m2, points, nbpoints, sv1_pos, sv2_pos,max_distance, min_distance);
- DP *sv1 = points[sv1_pos];
- DP *sv2 = points[sv2_pos];
+ ph_selectvantagepoints(&m2, points, nbpoints, sv1_pos, sv2_pos);
+ DP *sv1 = NULL;
+ DP *sv2 = NULL;
+ if (sv1_pos >= 0) sv1 = points[sv1_pos];
+ if (sv2_pos >= 0) sv2 = points[sv2_pos];
/* if file pos is beyond pg size*/
if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,&m2) > m->pgsize)
- return (FileIndex*)PH_ERRPGSIZE;
+ return PH_ERRSMPGSIZE;
ph_save_datapoint(sv1, &m2);
-
+
/*if file pos is beyond pg size */
- if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,&m2) > m->pgsize)
- return (FileIndex*)PH_ERRPGSIZE;
+ if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv2,&m2) > m->pgsize)
+ return PH_ERRSMPGSIZE;
ph_save_datapoint(sv2, &m2);
@@ -1486,6 +1729,14 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
/* write d1, d2 */
d1 = hashdist(sv1, points[i]);
d2 = hashdist(sv2, points[i]);
+
+ if (level < PathLength){
+ points[i]->path[level] = d1;
+ }
+ if (level+1 < PathLength){
+ points[i]->path[level+1] = d2;
+ }
+
memcpy(&(m2.buf[m2.file_pos & offset_mask]), &d1, sizeof(float));
m2.file_pos += sizeof(float);
memcpy(&(m2.buf[m2.file_pos & offset_mask]), &d2, sizeof(float));
@@ -1496,8 +1747,8 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
m2.file_pos = last_pos;
/* if file pos is beyond pg size */
- if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,&m2) > m->pgsize)
- return (FileIndex*)PH_ERRPGSIZE;
+ if ((m->file_pos & offset_mask) + ph_sizeof_dp(points[i],&m2) > m->pgsize)
+ return PH_ERRSMPGSIZE;
dp_pos = ph_save_datapoint(points[i], &m2);
last_pos = m2.file_pos;
@@ -1508,49 +1759,36 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
}
if (msync(m2.buf, m->pgsize, MS_SYNC) < 0){
- free(pOffset);
- close(m2.fd);
- return NULL;
+ close(m2.fd);
+ return PH_ERRMSYNC;
}
munmap(m2.buf, m->pgsize);
if (ftruncate(m2.fd, end_pos) < 0){
- free(pOffset);
close(m2.fd);
- return NULL;
+ return PH_ERRFILETRUNC;
}
-
close(m2.fd);
- } else {
+ } else { /* internal node */
off_t pgsize = m->pgsize;
-
pOffset->fileno = 0;
pOffset->offset = m->file_pos;
-
off_t orig_pos = m->file_pos;
off_t end_pos;
-
+ char *tmpbuf;
if ((level > 0) && (saveall_flag == 1)){ /* append new page to mainfile, unmap/map to it */
-
- if (msync(m->buf, pgsize ,MS_SYNC) < 0){
- free(pOffset);
- return NULL;
- }
-
- munmap(m->buf, pgsize);
-
+ tmpbuf = m->buf;
+
m->file_pos = lseek(m->fd, 0, SEEK_END);
end_pos = m->file_pos + pgsize;
if (ftruncate(m->fd, m->file_pos + pgsize) < 0){
- free(pOffset);
- return NULL;
+ return PH_ERRFILETRUNC;
}
off_t pa_offset = m->file_pos & page_mask;
m->buf = (char*)mmap(NULL,pgsize,PROT_READ|PROT_WRITE,MAP_SHARED,m->fd,pa_offset);
if (m->buf == MAP_FAILED){
- free(pOffset);
- return NULL;
+ return PH_ERRMMAP;
}
pOffset->offset = m->file_pos;
}
@@ -1560,8 +1798,7 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
/* choose vantage points, sv1, sv2 */
int sv1_pos, sv2_pos;
- float max_distance, min_distance;
- ph_selectvantagepoints(m, points, nbpoints, sv1_pos, sv2_pos, max_distance, min_distance);
+ ph_selectvantagepoints(m, points, nbpoints, sv1_pos, sv2_pos);
DP *sv1 = points[sv1_pos];
DP *sv2 = points[sv2_pos];
@@ -1569,21 +1806,36 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
/* save sv1, sv2 */
/* check that file_pos does not exceed pgsize */
- if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,m) > m->pgsize)
- return (FileIndex*)PH_ERRPGSIZE;
+ if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,m) > m->pgsize){
+ return PH_ERRSMPGSIZE;
+ }
ph_save_datapoint(sv1, m);
/* check the file_os does not exceed pgsize */
- if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,m) > m->pgsize)
- return (FileIndex*)PH_ERRPGSIZE;
+ if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv2,m) > m->pgsize){
+ return PH_ERRSMPGSIZE;
+ }
ph_save_datapoint(sv2, m);
+ float max_distance = 0.0f, min_distance = (float)INT_MAX;
+ for (int i=0;i<nbpoints;i++){
+ if (i != sv1_pos){
+ float d = hashdist(sv1,points[i]);
+ if (d > max_distance){
+ max_distance = d;
+ }
+ if (d < min_distance){
+ min_distance = d;
+ }
+ }
+ }
+
/* 1st tier pivots, M1, derived from the distance of each point from sv1*/
float step = (max_distance - min_distance)/BranchFactor;
- if (step <= 0.5){
- return (FileIndex*)PH_ERRDIST;
+ if (step <= 0.001){
+ return PH_ERRDIST;
}
float incr = step;
@@ -1591,8 +1843,7 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
float *M1 = (float*)malloc(LengthM1*sizeof(float));
float *M2 = (float*)malloc(LengthM2*sizeof(float));
if (!M1 || !M2){
- free(pOffset);
- return NULL;
+ return PH_ERRMEMALLOC;
}
@@ -1605,31 +1856,28 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
/*set up 1st tier sorting bins - contain pointers to DP points[] param so that we only
move pointers, not the actual datapoints */
- DP ***bins = (DP***)malloc(BranchFactor*sizeof(DP***));
+ DP ***bins = (DP***)malloc(BranchFactor*sizeof(DP**));
if (!bins){
- free(pOffset);
free(M1);
free(M2);
- return NULL;
+ return PH_ERRMEMALLOC;
}
int *mlens = (int*)calloc(BranchFactor, sizeof(int)); /*no. points in each bin */
if (!mlens){
- free(pOffset);
free(M1);
free(M2);
free(bins);
- return NULL;
+ return PH_ERRMEMALLOC;
}
for (int i=0;i<BranchFactor;i++){
- bins[i] = (DP**)malloc(Np*sizeof(DP**)); /*Np should be more than enough */
+ bins[i] = (DP**)malloc(Np*sizeof(DP*)); /*Np should be more than enough */
if (!bins[i]){
- free(pOffset);
free(M1);
free(M2);
free(bins);
free(mlens);
- return NULL;
+ return PH_ERRMEMALLOC;
}
}
@@ -1667,41 +1915,38 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
/* set up 2nd tier sorting bins */
/* each row from bins to be sorted into bins2, each in turn */
- DP ***bins2 = (DP***)malloc(BranchFactor*sizeof(DP***));
+ DP ***bins2 = (DP***)malloc(BranchFactor*sizeof(DP**));
if (!bins2){
- free(pOffset);
free(M1);
free(M2);
free(bins);
free(mlens);
- return NULL;
+ return PH_ERRMEMALLOC;
}
int *mlens2 = (int*)calloc(BranchFactor, sizeof(int)); /*number points in each bin */
if (!mlens2){
- free(pOffset);
free(M1);
free(M2);
free(bins);
free(bins2);
free(mlens);
- return NULL;
+ return PH_ERRMEMALLOC;
}
for (int i=0;i<BranchFactor;i++){
- bins2[i] = (DP**)malloc(Np*sizeof(DP**)); /* Np is more than enough */
+ bins2[i] = (DP**)malloc(Np*sizeof(DP*)); /* Np is more than enough */
if (!bins2[i]){
- free(pOffset);
free(M1);
free(M2);
free(bins);
free(mlens);
free(bins2);
free(mlens2);
- return NULL;
+ return PH_ERRMEMALLOC;
}
}
-
+ FileIndex ChildIndex;
off_t m2_pos = m->file_pos; /* start of M2 pivots */
off_t child_pos = m2_pos + LengthM2*sizeof(float); /*pos where child offsets are written*/
off_t last_pos = child_pos + Fanout*(sizeof(uint8_t) + sizeof(off_t)); /* last pos in
@@ -1713,22 +1958,14 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
for (int j=0;j < BranchFactor;j++){ /* reset the lengths to 0 */
mlens2[j] = 0;
}
- float *dist_temp = (float*)realloc(distance_vector, row_len*sizeof(float));
- if (!dist_temp){
- free(pOffset);
- free(M1);
- free(M2);
- free(bins);
- free(mlens);
- free(bins2);
- free(mlens2);
- return NULL;
+ float *distance_vector = (float*)malloc( row_len*sizeof(float));
+ if (!distance_vector){
+ goto cleanup;
}
- distance_vector = dist_temp;
/* 2nd tier pivots M2[], for row */
- max_distance = 0;
- min_distance = INT_MAX;
+ max_distance = 0.0f;
+ min_distance = (float)INT_MAX;
for (int j=0;j<row_len;j++){
distance_vector[j] = hashdist(sv2, bins[i][j]);
if ( distance_vector[j] > max_distance)
@@ -1778,61 +2015,53 @@ FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_f
*/
/* save child nodes */
- FileIndex *pChild = NULL;
for (int j=0;j<BranchFactor;j++){
m->file_pos = last_pos;
- pChild = _ph_save_mvptree(m, bins2[j], mlens2[j], saveall_flag, level+2);
- if (pChild){ /* write filenumber and offset of child node */
+ ret = _ph_save_mvptree(m, bins2[j], mlens2[j], saveall_flag, level+2, &ChildIndex);
+ if (ret == PH_SUCCESS){ /* write filenumber and offset of child node */
last_pos = m->file_pos;
m->file_pos = child_pos;
- memcpy(&(m->buf[m->file_pos & offset_mask]),&(pChild->fileno),sizeof(uint8_t));
+ memcpy(&(m->buf[m->file_pos & offset_mask]),&(ChildIndex.fileno),sizeof(uint8_t));
m->file_pos++;
- memcpy(&(m->buf[m->file_pos & offset_mask]), &(pChild->offset),sizeof(off_t));
+ memcpy(&(m->buf[m->file_pos & offset_mask]), &(ChildIndex.offset),sizeof(off_t));
m->file_pos += sizeof(off_t);
child_pos = m->file_pos;
- } else { /* child node is null */
- last_pos = m->file_pos;
- m->file_pos = child_pos;
- uint8_t emptyfileno = 0;
- memcpy(&(m->buf[m->file_pos & offset_mask]), &emptyfileno, sizeof(uint8_t));
- m->file_pos++;
- off_t emptyoffset = 0;
- memcpy(&(m->buf[m->file_pos & offset_mask]), &emptyoffset, sizeof(off_t));
- m->file_pos += sizeof(off_t);
- child_pos = m->file_pos;
+ } else { /* child not saved successfully */
+ goto cleanup;
}
}
m->file_pos = last_pos;
+ free(distance_vector);
}
/* remap to orig_pos */
if ((level > 0) && (saveall_flag == 1)){
/* unmap/remap to page with original position */
if (msync(m->buf, pgsize, MS_SYNC) < 0){
- free(pOffset);
+ ret = PH_ERRMSYNC;
goto cleanup;
}
munmap(m->buf, pgsize);
- m->buf=(char*)mmap(NULL,pgsize,PROT_WRITE|PROT_READ,MAP_SHARED,
- m->fd,orig_pos & page_mask);
- if (m->buf == MAP_FAILED){
- free(pOffset);
- goto cleanup;
- }
+ m->buf = tmpbuf;
m->file_pos = orig_pos;
}
/* cleanup */
cleanup:
- free(distance_vector);
+ for (int i=0;i<m->branchfactor;i++){
+ free(bins[i]);
+ }
free(bins);
+ for (int i=0;i<m->branchfactor;i++){
+ free(bins2[i]);
+ }
free(bins2);
free(mlens);
free(mlens2);
free(M1);
free(M2);
}
- return pOffset;
+ return ret;
}
@@ -1854,17 +2083,24 @@ MVPRetCode ph_save_mvptree(MVPFile *m, DP **points, int nbpoints){
return PH_ERRARG;
}
+ for (int i=0;i<nbpoints;i++){
+ points[i]->path = (float*)malloc((m->pathlength)*sizeof(float));
+ if (points[i]->path == NULL){
+ return PH_ERRMEMALLOC;
+ }
+ }
+
/* open main file */
char mainfile[256];
snprintf(mainfile, sizeof(mainfile),"%s.mvp", m->filename);
m->fd = open(mainfile, O_CREAT|O_RDWR|O_TRUNC, 00755);
if (m->fd < 0){
- return PH_ERRFILE;
+ return PH_ERRFILEOPEN;
}
/* enlarge to page*/
m->file_pos = 0;
if (ftruncate(m->fd, m->pgsize) < 0){
- return PH_ERRFILE;
+ return PH_ERRFILETRUNC;
}
m->buf = (char*)mmap(NULL,m->pgsize,PROT_READ|PROT_WRITE,MAP_SHARED,m->fd,m->file_pos);
@@ -1906,9 +2142,9 @@ MVPRetCode ph_save_mvptree(MVPFile *m, DP **points, int nbpoints){
m->file_pos = HeaderSize;
- if (_ph_save_mvptree(m, points, nbpoints, 1, 0) == NULL){
- return PH_ERRSAVEMVP;
- }
+ FileIndex NodeIndex;
+ MVPRetCode ret = _ph_save_mvptree(m, points, nbpoints, 1, 0, &NodeIndex);
+
memcpy(&m->buf[nbfiles_pos],&m->nbdbfiles, sizeof(uint8_t));
@@ -1917,10 +2153,13 @@ MVPRetCode ph_save_mvptree(MVPFile *m, DP **points, int nbpoints){
}
munmap(m->buf, m->pgsize);
-
close(m->fd);
- return PH_SUCCESS;
+ for (int i=0;i<nbpoints;i++){
+ free(points[i]->path);
+ }
+
+ return ret;
}
@@ -1929,14 +2168,15 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
uint8_t ntype;
off_t offset_mask, page_mask;
+ MVPRetCode ret = PH_SUCCESS;
if ( (!m) || (!new_dp) ||(level < 0))
- return PH_ERRARG;
+ return PH_ERRNULLARG;
hash_compareCB hashdist = m->hashdist;
if (!hashdist)
- return PH_ERRARG;
+ return PH_ERRNULLARG;
offset_mask = m->pgsize - 1;
page_mask = ~(m->pgsize - 1);
@@ -1959,8 +2199,14 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
float d1 = hashdist(sv1,new_dp);
float d2 = hashdist(sv2,new_dp);
+ if (level < m->pathlength){
+ new_dp->path[level] = d1;
+ }
+ if (level+1 < m->pathlength){
+ new_dp->path[level+1] = d2;
+ }
off_t curr_pos, new_pos, point_pos;
- if (Np == 0){
+ if (Np == 0){ /* add new point into empty leaf */
memcpy(&m->buf[m->file_pos & offset_mask], &d1, sizeof(float));
m->file_pos += sizeof(float);
memcpy(&m->buf[m->file_pos & offset_mask], &d2, sizeof(float));
@@ -1971,8 +2217,8 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
m->file_pos = offset_start+(m->leafcapacity)*(2*sizeof(float)+sizeof(off_t));
- if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,m) > m->pgsize)
- return PH_ERRPGSIZE;
+ if ((m->file_pos & offset_mask) + ph_sizeof_dp(new_dp,m) > m->pgsize)
+ return PH_ERRSMPGSIZE;
new_pos = ph_save_datapoint(new_dp, m);
@@ -1982,7 +2228,7 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
Np++;
memcpy(&m->buf[Np_pos & offset_mask], &Np, sizeof(uint8_t));
- } else if (Np < m->leafcapacity){
+ } else if (Np < m->leafcapacity){ /* add new point into leaf */
m->file_pos += (Np-1)*(2*sizeof(float)+ sizeof(off_t));
m->file_pos += 2*sizeof(float);
memcpy(&point_pos,&m->buf[m->file_pos & offset_mask], sizeof(off_t));
@@ -1999,8 +2245,8 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
m->file_pos += byte_len;
- if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,m) > m->pgsize)
- return PH_ERRPGSIZE;
+ if ((m->file_pos & offset_mask) + ph_sizeof_dp(new_dp,m) > m->pgsize)
+ return PH_ERRSMPGSIZE;
new_pos = ph_save_datapoint(new_dp, m);
memcpy(&m->buf[curr_pos & offset_mask],&d1, sizeof(float));
@@ -2012,12 +2258,12 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
Np++;
memcpy(&m->buf[Np_pos & offset_mask], &Np, sizeof(uint8_t));
- } else {
- DP **points = (DP**)malloc((Np+3)*sizeof(DP**));
+ } else { /* convert leaf node into internal node */
+ DP **points = (DP**)malloc((Np+3)*sizeof(DP*));
points[0] = sv1;
points[1] = sv2;
- for (int i=2;i < Np+2;i++){
+ for (int i=2;i < Np+2;i++){ /* read in all points */
m->file_pos += 2*sizeof(float);
memcpy(&point_pos,&m->buf[m->file_pos & offset_mask], sizeof(off_t));
m->file_pos += sizeof(off_t);
@@ -2029,38 +2275,45 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
}
points[Np+2] = new_dp;
m->file_pos = start_pos;
- if (!_ph_save_mvptree(m, points, Np+3, 0, level+2)){
- free(points);
- free(sv1);
- free(sv2);
- return PH_ERRSAVEMVP;
+ FileIndex ChildIndex;
+ ret = _ph_save_mvptree(m, points, Np+3, 0, level+2, &ChildIndex);
+ for (int i=2;i<Np+2;i++){
+ free(points[i]->id);
+ free(points[i]->path);
+ free(points[i]->hash);
+ ph_free_datapoint(points[i]);
}
- free(points);
+ free(points);
}
+ free(sv2->id);
+ free(sv2->path);
+ free(sv2->hash);
+ ph_free_datapoint(sv2);
} else { /* put new point into sv2 pos */
m->file_pos = start_pos;
ntype = 0;
memcpy(&m->buf[m->file_pos & offset_mask], &ntype, sizeof(uint8_t));
m->file_pos++;
-
if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,m) > m->pgsize)
- return PH_ERRPGSIZE;
+ return PH_ERRSMPGSIZE;
ph_save_datapoint(sv1, m);
if ((m->file_pos & offset_mask) + ph_sizeof_dp(sv1,m) > m->pgsize)
- return PH_ERRPGSIZE;
+ return PH_ERRSMPGSIZE;
ph_save_datapoint(new_dp,m);
Np = 0;
memcpy(&m->buf[m->file_pos & offset_mask], &Np, sizeof(uint8_t));
m->file_pos++;
}
+ free(sv1->id);
+ free(sv1->path);
+ free(sv1->hash);
ph_free_datapoint(sv1);
- ph_free_datapoint(sv2);
}
- else {
+ else { /* save new dp into sv1 position */
m->file_pos = start_pos;
ntype = 0;
memcpy(&m->buf[m->file_pos & offset_mask], &ntype, sizeof(uint8_t));
@@ -2080,11 +2333,11 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
float *M1 = (float*)malloc(LengthM1*sizeof(float));
if (!M1){
- return PH_ERRMEM;
+ return PH_ERRMEMALLOC;
}
float *M2 = (float*)malloc(LengthM2*sizeof(float));
if (!M2){
- return PH_ERRMEM;
+ return PH_ERRMEMALLOC;
}
memcpy(M1, &m->buf[m->file_pos & offset_mask], LengthM1*sizeof(float));
@@ -2095,10 +2348,17 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
if (level < m->pathlength)
new_dp->path[level] = d1;
- if (level < m->pathlength - 1)
+ if (level+1 < m->pathlength)
new_dp->path[level+1] = d2;
+ free(sv1->id);
+ free(sv1->path);
+ free(sv1->hash);
ph_free_datapoint(sv1);
+
+ free(sv2->id);
+ free(sv2->path);
+ free(sv2->hash);
ph_free_datapoint(sv2);
int pivot1, pivot2;
@@ -2106,7 +2366,7 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
off_t child_pos, curr_pos;
off_t start_pos = m->file_pos;
off_t orig_pos;
- MVPRetCode retcode;
+ MVPFile m2;
/* check <= each M1 pivot */
for (pivot1=0;pivot1 < LengthM1;pivot1++){
if (d1 <= M1[pivot1]){
@@ -2122,39 +2382,56 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
memcpy(&child_pos,&m->buf[m->file_pos & offset_mask], sizeof(off_t));
m->file_pos += sizeof(off_t);
- /* save position and remap to new file/position */
- orig_pos = m->file_pos;
- MVPFile *m2 = _ph_map_mvpfile(filenumber,child_pos,m);
- if (m2){
- retcode = _ph_add_mvptree(m2, new_dp, level+2);
- if (retcode != PH_SUCCESS){
- return retcode;
+ if ((filenumber == 0) && (child_pos == 0)){
+ FileIndex child_offset;
+ ret = _ph_save_mvptree(m, &new_dp, 1, 0, level+2, &child_offset);
+ if (ret == PH_SUCCESS){
+ memcpy(&m->buf[curr_pos++ & offset_mask], &(child_offset.fileno), sizeof(uint8_t));
+ memcpy(&m->buf[curr_pos & offset_mask], &(child_offset.offset), sizeof(off_t));
}
+ } else {
+ /* save position and remap to new file/position */
+ ret = _ph_map_mvpfile(filenumber,child_pos,m,&m2);
+ if (ret == PH_SUCCESS){
+ ret = _ph_add_mvptree(&m2, new_dp, level+2);
+ if (ret != PH_SUCCESS){
+ goto addcleanup;
+ }
+ ret = _ph_unmap_mvpfile(filenumber, orig_pos, m, &m2);
+ }
}
- _ph_unmap_mvpfile(filenumber, orig_pos, m, m2);
-
+ goto addcleanup;
}
- }
- /* check > last M2 pivot */
- if (d2 > M2[LengthM1-1+pivot1*LengthM1]){
- curr_pos = start_pos + (m->branchfactor-1+pivot1*m->branchfactor)*(sizeof(uint8_t)+sizeof(off_t));
- m->file_pos = curr_pos;
- memcpy(&filenumber,&m->buf[m->file_pos & offset_mask], sizeof(uint8_t));
- m->file_pos++;
- memcpy(&child_pos,&m->buf[m->file_pos & offset_mask], sizeof(off_t));
- m->file_pos += sizeof(off_t);
+
+ /* check > last M2 pivot */
+ if (d2 > M2[LengthM1-1+pivot1*LengthM1]){
+ curr_pos = start_pos + (m->branchfactor-1+pivot1*m->branchfactor)*(sizeof(uint8_t)+sizeof(off_t));
+ m->file_pos = curr_pos;
+ memcpy(&filenumber,&m->buf[m->file_pos & offset_mask], sizeof(uint8_t));
+ m->file_pos++;
+ memcpy(&child_pos,&m->buf[m->file_pos & offset_mask], sizeof(off_t));
+ m->file_pos += sizeof(off_t);
- orig_pos = m->file_pos;
- MVPFile *m2 = _ph_map_mvpfile(filenumber, child_pos, m);
- if (m2){
- retcode = _ph_add_mvptree(m2, new_dp, level+2);
- if (retcode != PH_SUCCESS){
- return retcode;
+ if ((filenumber == 0) && (child_pos == 0)){
+ FileIndex child_offset;
+ ret = _ph_save_mvptree(m, &new_dp, 1, 0, level+2, &child_offset);
+ if (ret == PH_SUCCESS){
+ memcpy(&m->buf[curr_pos++ & offset_mask], &child_offset.fileno, sizeof(uint8_t));
+ memcpy(&m->buf[curr_pos & offset_mask], &child_offset.offset, sizeof(off_t));
+ }
+ } else {
+ ret = _ph_map_mvpfile(filenumber, child_pos, m, &m2);
+ if (ret == PH_SUCCESS){
+ ret = _ph_add_mvptree(&m2, new_dp, level+2);
+ if (ret != PH_SUCCESS){
+ goto addcleanup;
+ }
+ ret = _ph_unmap_mvpfile(filenumber,orig_pos, m, &m2);
+ }
}
+ goto addcleanup;
}
- _ph_unmap_mvpfile(filenumber,orig_pos, m, m2);
}
-
}
}
/* check > last M1 pivot */
@@ -2168,16 +2445,25 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
m->file_pos++;
memcpy(&child_pos,&m->buf[m->file_pos & offset_mask], sizeof(off_t));
m->file_pos += sizeof(off_t);
-
- orig_pos = m->file_pos;
- MVPFile *m2 = _ph_map_mvpfile(filenumber,child_pos,m);
- if (m2){
- retcode = _ph_add_mvptree(m2,new_dp,level+2);
- if (retcode != PH_SUCCESS){
- return retcode;
+
+ if ((filenumber == 0) && (child_pos == 0)){
+ FileIndex child_offset;
+ ret = _ph_save_mvptree(m, &new_dp, 1, 0, level+2, &child_offset);
+ if (ret == PH_SUCCESS){
+ memcpy(&m->buf[curr_pos++ & offset_mask],&child_offset.fileno,sizeof(uint8_t));
+ memcpy(&m->buf[curr_pos & offset_mask],&child_offset.offset,sizeof(off_t));
+ }
+ } else {
+ ret = _ph_map_mvpfile(filenumber,child_pos,m, &m2);
+ if (ret == PH_SUCCESS){
+ ret = _ph_add_mvptree(&m2,new_dp,level+2);
+ if (ret != PH_SUCCESS){
+ goto addcleanup;
+ }
+ ret = _ph_unmap_mvpfile(filenumber, orig_pos, m, &m2);
}
}
- _ph_unmap_mvpfile(filenumber, orig_pos, m, m2);
+ goto addcleanup;
}
}
@@ -2190,36 +2476,48 @@ MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level){
memcpy(&child_pos, &m->buf[m->file_pos & offset_mask], sizeof(off_t));
m->file_pos += sizeof(off_t);
- orig_pos = m->file_pos;
- MVPFile *m2 = _ph_map_mvpfile(filenumber, child_pos, m);
- if (m2){
- retcode = _ph_add_mvptree(m2, new_dp, level+2);
- if (retcode != PH_SUCCESS){
- return retcode;
+ if ((filenumber == 0)&&(child_pos == 0)){
+ FileIndex child_offset;
+ ret = _ph_save_mvptree(m, &new_dp, 1, 0, level+2, &child_offset);
+ if (ret == PH_SUCCESS){
+ memcpy(&m->buf[curr_pos++ & offset_mask],&child_offset.fileno,sizeof(uint8_t));
+ memcpy(&m->buf[curr_pos & offset_mask],&child_offset.offset,sizeof(off_t));
}
+ } else {
+ ret = _ph_map_mvpfile(filenumber, child_pos, m, &m2);
+ if (ret == PH_SUCCESS){
+ ret = _ph_add_mvptree(&m2, new_dp, level+2);
+ if (ret != PH_SUCCESS){
+ goto addcleanup;
+ }
+ ret = _ph_unmap_mvpfile(filenumber, orig_pos, m, &m2);
+ }
+
}
- _ph_unmap_mvpfile(filenumber, orig_pos, m, m2);
+ goto addcleanup;
}
}
+
+addcleanup:
free(M1);
free(M2);
} else {
- return PH_ERRNTYPE;
+ ret = PH_ERRNTYPE;
}
- return PH_SUCCESS;
+ return ret;
}
-int ph_add_mvptree(MVPFile *m, DP **points, int nbpoints){
-
+MVPRetCode ph_add_mvptree(MVPFile *m, DP **points, int nbpoints, int &nbsaved){
+ nbsaved = 0;
/* open main file */
char mainfile[256];
snprintf(mainfile, sizeof(mainfile),"%s.mvp", m->filename);
m->fd = open(mainfile, O_RDWR);
if (m->fd < 0){
- return -1;
+ return PH_ERRFILEOPEN;
}
/*use host pg size until pg size of file can be read */
@@ -2229,7 +2527,7 @@ int ph_add_mvptree(MVPFile *m, DP **points, int nbpoints){
m->file_pos = 0;
m->buf = (char*)mmap(NULL,m->pgsize,PROT_READ|PROT_WRITE,MAP_SHARED,m->fd,m->file_pos);
if (m->buf == MAP_FAILED){
- return -1;
+ return PH_ERRMMAP;
}
madvise(m->buf, m->pgsize,MADV_SEQUENTIAL);
@@ -2265,8 +2563,15 @@ int ph_add_mvptree(MVPFile *m, DP **points, int nbpoints){
m->file_pos = HeaderSize;
- m->pgsize = int_pgsize;
-
+ m->filenumber = 0;
+
+ for (int i=0;i < nbpoints;i++){
+ points[i]->path = (float*)malloc((m->pathlength)*sizeof(float));
+ if (points[i]->path == NULL){
+ return PH_ERRMEMALLOC;
+ }
+ }
+
/* remap to true pg size used in making file */
#ifdef HAVE_REMAP
m->buf = (char*)mremap(m->buf, m->pgsize, int_pgsize, MREMAP_MAYMOVE);
@@ -2275,32 +2580,38 @@ int ph_add_mvptree(MVPFile *m, DP **points, int nbpoints){
m->buf = (char*)mmap(m->buf, int_pgsize, PROT_READ|PROT_WRITE, MAP_SHARED, m->fd, 0);
#endif
if (m->buf == MAP_FAILED){
- return -1;
+ return PH_ERRMMAP;
}
- int nbsaved = 0;
- int retval;
+ m->pgsize = int_pgsize;
+
+ MVPRetCode retval = PH_SUCCESS;
for (int i=0;i<nbpoints;i++){
m->file_pos = HeaderSize;
retval = _ph_add_mvptree(m, points[i], 0);
- if (retval != 0){
+ if (retval != PH_SUCCESS){
+ fprintf(stderr,"unable to add %d, retcode %d\n", i, retval);
continue;
}
nbsaved++;
}
/* save new nbdbfiles if new files added */
- memcpy(&m->nbdbfiles, &m->buf[m->file_pos++], 1);
+ memcpy(&m->buf[nb_pos], &m->nbdbfiles, 1);
if (msync(m->buf, m->pgsize, MS_SYNC) < 0){
- return -1;
+ return PH_ERRMSYNC;
}
munmap(m->buf, m->pgsize);
-
close(m->fd);
- return nbsaved;
+ for (int i=0;i<nbpoints;i++){
+ free(points[i]->path);
+ }
+
+
+ return retval;
}
diff --git a/src/pHash.h b/src/pHash.h
index dfdfd96..2de03e2 100644
--- a/src/pHash.h
+++ b/src/pHash.h
@@ -18,7 +18,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
*/
@@ -41,6 +41,10 @@
#include <stdint.h>
#include "CImg.h"
+#ifdef HAVE_PTHREAD
+#include <pthread.h>
+#endif
+
using namespace cimg_library;
using namespace std;
@@ -67,19 +71,23 @@ extern "C" {
const int MaxFileSize = (1<<30); /* 1GB file size limit (for mvp files) */
const off_t HeaderSize = 64; /* header size for mvp file */
-
const char mvptag[] = "pHashMVPfile2009";
typedef enum ph_mvp_retcode {
PH_SUCCESS = 0, /* success */
PH_ERRPGSIZE, /* page size error */
- PH_ERRFILE, /* file operations */
+ PH_ERRSMPGSIZE,
+ PH_ERRFILEOPEN, /* file open err */
+ PH_ERRFILETRUNC,
+ PH_ERRFILESEEK,
PH_ERRMMAP, /* mmap'ing error */
PH_ERRMSYNC, /* msync error */
PH_ERRTRUNC, /* error truncating file */
PH_ERRSAVEMVP, /* could not save mvp file */
- PH_ERRARG, /* null arg */
- PH_ERRMEM, /* mem alloc error - not enough available memory */
+ PH_ERRARG, /* general arg err*/
+ PH_ERRNULLARG, /* null arg */
+ PH_ERRMEM, /* general memory error */
+ PH_ERRMEMALLOC, /* mem alloc error */
PH_ERRNTYPE, /* unrecognized node type */
PH_ERRCAP, /* more results found than can be supported in ret array */
PH_ERRFILETYPE, /*unrecognized file type */
@@ -105,18 +113,25 @@ typedef struct ph_datapoint {
char *id;
void *hash;
float *path;
- uint16_t hash_length;
+ uint32_t hash_length;
uint8_t hash_type;
}DP;
+typedef struct ph_slice {
+ DP **hash_p;
+ int n;
+ void *hash_params;
+} slice;
+
/* call back function for mvp tree functions - to performa distance calc.'s*/
typedef float (*hash_compareCB)(DP *pointA, DP *pointB);
typedef struct ph_mvp_file {
- const char *filename; /* name of db to use */
+ char *filename; /* name of db to use */
char *buf;
off_t file_pos;
int fd;
+ uint8_t filenumber;
uint8_t nbdbfiles;
uint8_t branchfactor; /*branch factor of tree, M(=2)*/
@@ -140,14 +155,7 @@ typedef struct ph_mvp_file {
/* convenience function to set var's of mvp tree */
-void ph_mvp_init(MVPFile *m){
- m->branchfactor = 2;
- m->pathlength = 5;
- m->leafcapacity = 23;
- m->pgsize = sysconf(_SC_PAGE_SIZE); /* use host page size */
- return;
-}
-
+void ph_mvp_init(MVPFile *m);
/*! /brief Radon Projection info
*/
@@ -191,10 +199,14 @@ typedef struct ph_match{
uint32_t length; /*length of match between 2 files */
} TxtMatch;
+#ifdef HAVE_PTHREAD
+int ph_num_threads();
+#endif
+
/* /brief alloc a single data point
* allocates path array, does nto set id or path
*/
- DP* ph_malloc_datapoint(int hashtype, int pathlength);
+ DP* ph_malloc_datapoint(int hashtype);
/** /brief free a datapoint and its path
*
@@ -303,10 +315,14 @@ static CImg<float>* ph_dct_matrix(const int N);
*/
int ph_dct_imagehash(const char* file,ulong64 &hash);
-
+#ifdef HAVE_PTHREAD
+DP** ph_dct_image_hashes(char *files[], int count, int threads = 0);
+#endif
static CImgList<uint8_t>* ph_getKeyFramesFromVideo(const char *filename);
-ulong64* ph_dct_videohash(const char *filename, int &Lenght);
+ulong64* ph_dct_videohash(const char *filename, int &Length);
+
+DP** ph_dct_video_hashes(char *files[], int count, int threads = 0);
double ph_dct_videohash_dist(ulong64 *hashA, int N1, ulong64 *hashB, int N2, int threshold=21);
@@ -334,7 +350,7 @@ DP** ph_read_imagehashes(const char *dirname,int capacity, int &count);
* /param lvl - int level of scale factor (default = 1)
* /return uint8_t array
**/
-uint8_t* ph_mh_imagehash(const char *filename, int &N, int alpha=2, int lvl = 1);
+uint8_t* ph_mh_imagehash(const char *filename, int &N, float alpha=2.0f, float lvl = 1.0f);
/** /brief count number bits set in given byte
* /param val - uint8_t byte value
@@ -386,14 +402,14 @@ off_t ph_save_datapoint(DP *new_dp, MVPFile *m);
* /param m - MVPFile
* /return MVPFile - ptr to new struct containing the mmap info
**/
-MVPFile* _ph_map_mvpfile(uint8_t filenumber, off_t offset, MVPFile *m);
+MVPRetCode _ph_map_mvpfile(uint8_t filenumber, off_t offset, MVPFile *m, MVPFile *m2);
/** /brief unmap/map from m2 to m
* /param filenumber - uint8_t filenumber of m2
* /param orig_pos = off_t offset into original file in m.
* /return void
**/
-void _ph_unmap_mvpfile(uint8_t filenumber, off_t orig_pos, MVPFile *m, MVPFile *m2);
+MVPRetCode _ph_unmap_mvpfile(uint8_t filenumber, off_t orig_pos, MVPFile *m, MVPFile *m2);
/**
* callback function for dct image hash use in mvptree structure.
@@ -406,12 +422,12 @@ float hammingdistance(DP *pntA, DP *pntB);
* /param knearest - int capacity of results array.
* /param radius - float value of radius of values to consider
* /param results - DP array of points of result
- * /param count - int* number of results found (out)
+ * /param nbfound - int number of results found (out)
* /param level - int value to track recursion depth.
* /return MVPRetCode
**/
-static MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
- DP **results, int *count, int level);
+static MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
+ float threshold, DP **results, int &nbfound, int level);
/** /brief query mvptree function
* /param m - MVPFile file state info
@@ -419,10 +435,10 @@ static MVPRetCode _ph_query_mvptree(MVPFile *m, DP *query, int knearest, float r
* /param knearest - int capacity of results array
* /param radius - float radius to consider in query
* /param results - DP** list of pointers to results found
- * /param count - int number of results found (out)
+ * /param nbfound - int number of results found (out)
**/
MVPRetCode ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
- DP **results, int *count);
+ float threshold, DP **results, int &nbfound);
/** /brief save dp points to a file (aux func)
* /param m - MVPFile state information of file
@@ -432,7 +448,7 @@ MVPRetCode ph_query_mvptree(MVPFile *m, DP *query, int knearest, float radius,
* /param level - int track recursion level
* /return FileIndex* - fileno and offset into file.
**/
-static FileIndex* _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_flag, int level);
+static MVPRetCode _ph_save_mvptree(MVPFile *m, DP **points, int nbpoints, int saveall_flag, int level,FileIndex *pOffset);
/** /brief save points to mvp file
* /param m - MVPFile state info of file
@@ -457,7 +473,7 @@ static MVPRetCode _ph_add_mvptree(MVPFile *m, DP *new_dp, int level);
/param nbpoints - int number of points
/return int - number of points added, neg for error
**/
-int ph_add_mvptree(MVPFile *m, DP **points, int nbpoints);
+MVPRetCode ph_add_mvptree(MVPFile *m, DP **points, int nbpoints, int &nbsaved);
/** /brief textual hash for file
* /param filename - char* name of file
@@ -740,4 +756,5 @@ static const ulong64 textkeys[256] = {
#ifdef __cplusplus
}
#endif
+
#endif
diff --git a/src/ph_fft.c b/src/ph_fft.c
index ef9b0da..ea355ba 100644
--- a/src/ph_fft.c
+++ b/src/ph_fft.c
@@ -18,7 +18,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
*/
diff --git a/src/ph_fft.h b/src/ph_fft.h
index 9b6ffc1..3e13077 100644
--- a/src/ph_fft.h
+++ b/src/ph_fft.h
@@ -18,7 +18,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
Evan Klinger - eklinger at phash.org
- David Starkweather - dstarkweather at phash.org
+ D Grant Starkweather - dstarkweather at phash.org
*/
--
debian-forensics/libphash
More information about the forensics-changes
mailing list