[Forensics-changes] [yara] 198/415: Implement multithreading support

Hilko Bengen bengen at moszumanska.debian.org
Thu Apr 3 05:43:05 UTC 2014


This is an automated email from the git hooks/post-receive script.

bengen pushed a commit to branch debian
in repository yara.

commit c50ebf3f61f800df2e256718a9d5c2af832163a5
Author: Victor M. Alvarez <plusvic at gmail.com>
Date:   Thu Sep 26 14:01:56 2013 +0000

    Implement multithreading support
---
 Makefile.am               |   4 +-
 libyara/compiler.c        |  15 +-
 libyara/exec.c            |  17 +-
 libyara/grammar.c         |  18 +-
 libyara/grammar.y         |  18 +-
 libyara/libyara.c         |  52 ++++-
 libyara/parser.c          |  33 ++--
 libyara/rules.c           | 162 ++++++++++-----
 libyara/yara.h            | 134 +++++++++----
 yara-python/yara-python.c |   2 +-
 yara.c                    | 489 ++++++++++++++++++++++++++++++----------------
 11 files changed, 634 insertions(+), 310 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 9e8466c..c2b19b7 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -4,12 +4,12 @@ AM_CFLAGS=-g -O3
 SUBDIRS = libyara
 DIST_SUBDIRS = libyara
 
-AM_CPPFLAGS = -I$(srcdir)/libyara -g -O0
+AM_CPPFLAGS = -I$(srcdir)/libyara -g -O3
 ACLOCAL_AMFLAGS=-I m4
 
 bin_PROGRAMS = yara yarac
 
-yara_SOURCES = yara.c
+yara_SOURCES = threading.c yara.c
 yara_LDADD = libyara/.libs/libyara.a
 
 yarac_SOURCES = yarac.c
diff --git a/libyara/compiler.c b/libyara/compiler.c
index 2ebf986..c2a7898 100644
--- a/libyara/compiler.c
+++ b/libyara/compiler.c
@@ -296,7 +296,10 @@ int _yr_compiler_set_namespace(
       return result;
 
     ns->name = ns_name;
-    ns->flags = 0;
+
+    for (i = 0; i < MAX_THREADS; i++)
+      ns->t_flags[i] = 0;
+
     compiler->namespaces_count++;
   }
 
@@ -351,7 +354,7 @@ int _yr_compiler_compile_rules(
 
   // Write a null rule indicating the end.
   memset(&null_rule, 0xFA, sizeof(RULE));
-  null_rule.flags = RULE_FLAGS_NULL;
+  null_rule.g_flags = RULE_GFLAGS_NULL;
 
   yr_arena_write_data(
       compiler->rules_arena,
@@ -508,7 +511,13 @@ int yr_compiler_get_rules(
     yara_rules->externals_list_head = rules_file_header->externals_list_head;
     yara_rules->automaton = rules_file_header->automaton;
     yara_rules->code_start = rules_file_header->code_start;
-    yara_rules->matches_arena = NULL;
+    yara_rules->threads_count = 0;
+
+    #if WIN32
+    yara_rules->mutex = CreateMutex(NULL, FALSE, NULL);
+    #else
+    pthread_mutex_init(&yara_rules->mutex, NULL);
+    #endif
 
     *rules = yara_rules;
   }
diff --git a/libyara/exec.c b/libyara/exec.c
index 0a47462..2a3021d 100644
--- a/libyara/exec.c
+++ b/libyara/exec.c
@@ -89,6 +89,7 @@ int yr_execute_code(
   int found;
   int count;
   int result;
+  int tidx = yr_get_tidx();
 
   while(1)
   {
@@ -295,7 +296,7 @@ int yr_execute_code(
       case RULE_PUSH:
         rule = *(RULE**)(ip + 1);
         ip += sizeof(uint64_t);
-        push(rule->flags & RULE_FLAGS_MATCH ? 1 : 0);
+        push(rule->t_flags[tidx] & RULE_TFLAGS_MATCH ? 1 : 0);
         break;
 
       case RULE_POP:
@@ -303,7 +304,7 @@ int yr_execute_code(
         rule = *(RULE**)(ip + 1);
         ip += sizeof(uint64_t);
         if (r1)
-          rule->flags |= RULE_FLAGS_MATCH;
+          rule->t_flags[tidx] |= RULE_TFLAGS_MATCH;
         break;
 
       case EXT_INT:
@@ -331,7 +332,7 @@ int yr_execute_code(
       case SFOUND:
         pop(r1);
         string = UINT64_TO_PTR(STRING*, r1);
-        push(string->flags & STRING_FLAGS_FOUND ? 1 : 0);
+        push(string->matches[tidx].tail != NULL ? 1 : 0);
         break;
 
       case SFOUND_AT:
@@ -345,7 +346,7 @@ int yr_execute_code(
         }
 
         string = UINT64_TO_PTR(STRING*, r2);
-        match = string->matches_list_head;
+        match = string->matches[tidx].head;
         found = 0;
 
         while (match != NULL)
@@ -380,7 +381,7 @@ int yr_execute_code(
         }
 
         string = UINT64_TO_PTR(STRING*, r3);
-        match = string->matches_list_head;
+        match = string->matches[tidx].head;
         found = FALSE;
 
         while (match != NULL && !found)
@@ -407,7 +408,7 @@ int yr_execute_code(
       case SCOUNT:
         pop(r1);
         string = UINT64_TO_PTR(STRING*, r1);
-        match = string->matches_list_head;
+        match = string->matches[tidx].head;
         found = 0;
         while (match != NULL)
         {
@@ -428,7 +429,7 @@ int yr_execute_code(
         }
 
         string = UINT64_TO_PTR(STRING*, r2);
-        match = string->matches_list_head;
+        match = string->matches[tidx].head;
         i = 1;
         found = FALSE;
 
@@ -458,7 +459,7 @@ int yr_execute_code(
         while (r1 != UNDEFINED)
         {
           string = UINT64_TO_PTR(STRING*, r1);
-          if (string->flags & STRING_FLAGS_FOUND)
+          if (string->matches[tidx].tail != NULL)
             found++;
           count++;
           pop(r1);
diff --git a/libyara/grammar.c b/libyara/grammar.c
index c205cd9..466bef7 100644
--- a/libyara/grammar.c
+++ b/libyara/grammar.c
@@ -1836,7 +1836,7 @@ yyreduce:
           compiler = yyget_extra(yyscanner);
 
           memset(&null_string, 0xFF, sizeof(STRING));
-          null_string.flags = STRING_FLAGS_NULL;
+          null_string.g_flags = STRING_GFLAGS_NULL;
 
           yr_arena_write_data(
               compiler->strings_arena,
@@ -1861,12 +1861,12 @@ yyreduce:
 
   case 14:
 #line 256 "grammar.y"
-    { (yyval.integer) = RULE_FLAGS_PRIVATE; }
+    { (yyval.integer) = RULE_GFLAGS_PRIVATE; }
     break;
 
   case 15:
 #line 257 "grammar.y"
-    { (yyval.integer) = RULE_FLAGS_GLOBAL; }
+    { (yyval.integer) = RULE_GFLAGS_GLOBAL; }
     break;
 
   case 16:
@@ -2045,7 +2045,7 @@ yyreduce:
     {
                         (yyval.string) = yr_parser_reduce_string_declaration(
                             yyscanner,
-                            (yyvsp[(4) - (4)].integer) | STRING_FLAGS_REGEXP,
+                            (yyvsp[(4) - (4)].integer) | STRING_GFLAGS_REGEXP,
                             (yyvsp[(1) - (4)].c_string),
                             (yyvsp[(3) - (4)].sized_string));
 
@@ -2061,7 +2061,7 @@ yyreduce:
     {
                         (yyval.string) = yr_parser_reduce_string_declaration(
                             yyscanner,
-                            STRING_FLAGS_HEXADECIMAL,
+                            STRING_GFLAGS_HEXADECIMAL,
                             (yyvsp[(1) - (3)].c_string),
                             (yyvsp[(3) - (3)].sized_string));
 
@@ -2084,22 +2084,22 @@ yyreduce:
 
   case 33:
 #line 436 "grammar.y"
-    { (yyval.integer) = STRING_FLAGS_WIDE; }
+    { (yyval.integer) = STRING_GFLAGS_WIDE; }
     break;
 
   case 34:
 #line 437 "grammar.y"
-    { (yyval.integer) = STRING_FLAGS_ASCII; }
+    { (yyval.integer) = STRING_GFLAGS_ASCII; }
     break;
 
   case 35:
 #line 438 "grammar.y"
-    { (yyval.integer) = STRING_FLAGS_NO_CASE; }
+    { (yyval.integer) = STRING_GFLAGS_NO_CASE; }
     break;
 
   case 36:
 #line 439 "grammar.y"
-    { (yyval.integer) = STRING_FLAGS_FULL_WORD; }
+    { (yyval.integer) = STRING_GFLAGS_FULL_WORD; }
     break;
 
   case 38:
diff --git a/libyara/grammar.y b/libyara/grammar.y
index 8733895..8aa206d 100644
--- a/libyara/grammar.y
+++ b/libyara/grammar.y
@@ -230,7 +230,7 @@ strings : /* empty */
           compiler = yyget_extra(yyscanner);
 
           memset(&null_string, 0xFF, sizeof(STRING));
-          null_string.flags = STRING_FLAGS_NULL;
+          null_string.g_flags = STRING_GFLAGS_NULL;
 
           yr_arena_write_data(
               compiler->strings_arena,
@@ -253,8 +253,8 @@ rule_modifiers : /* empty */                      { $$ = 0;  }
                ;
 
 
-rule_modifier : _PRIVATE_       { $$ = RULE_FLAGS_PRIVATE; }
-              | _GLOBAL_        { $$ = RULE_FLAGS_GLOBAL; }
+rule_modifier : _PRIVATE_       { $$ = RULE_GFLAGS_PRIVATE; }
+              | _GLOBAL_        { $$ = RULE_GFLAGS_GLOBAL; }
               ;
 
 
@@ -403,7 +403,7 @@ string_declaration  : _STRING_IDENTIFIER_ '=' _TEXTSTRING_ string_modifiers
                       {
                         $$ = yr_parser_reduce_string_declaration(
                             yyscanner,
-                            $4 | STRING_FLAGS_REGEXP,
+                            $4 | STRING_GFLAGS_REGEXP,
                             $1,
                             $3);
 
@@ -416,7 +416,7 @@ string_declaration  : _STRING_IDENTIFIER_ '=' _TEXTSTRING_ string_modifiers
                       {
                         $$ = yr_parser_reduce_string_declaration(
                             yyscanner,
-                            STRING_FLAGS_HEXADECIMAL,
+                            STRING_GFLAGS_HEXADECIMAL,
                             $1,
                             $3);
 
@@ -433,10 +433,10 @@ string_modifiers : /* empty */                              { $$ = 0;  }
                  ;
 
 
-string_modifier : _WIDE_        { $$ = STRING_FLAGS_WIDE; }
-                | _ASCII_       { $$ = STRING_FLAGS_ASCII; }
-                | _NOCASE_      { $$ = STRING_FLAGS_NO_CASE; }
-                | _FULLWORD_    { $$ = STRING_FLAGS_FULL_WORD; }
+string_modifier : _WIDE_        { $$ = STRING_GFLAGS_WIDE; }
+                | _ASCII_       { $$ = STRING_GFLAGS_ASCII; }
+                | _NOCASE_      { $$ = STRING_GFLAGS_NO_CASE; }
+                | _FULLWORD_    { $$ = STRING_GFLAGS_FULL_WORD; }
                 ;
 
 
diff --git a/libyara/libyara.c b/libyara/libyara.c
index cf41590..fe9f968 100644
--- a/libyara/libyara.c
+++ b/libyara/libyara.c
@@ -24,14 +24,20 @@ limitations under the License.
 #define snprintf _snprintf
 #endif
 
+#ifdef WIN32
+#else
+#include <pthread.h>
+#define PTHREADS
+#endif
 
 char isregexescapable[256];
 char isregexhashable[256];
 char isalphanum[256];
 char lowercase[256];
 
+pthread_key_t key;
 
-void yr_initialize()
+void yr_initialize(void)
 {
   int i;
 
@@ -73,17 +79,51 @@ void yr_initialize()
   isregexescapable['\\'] = TRUE;
 
   yr_heap_alloc();
-}
+  
+  #ifdef PTHREADS
+  pthread_key_create(&key, NULL);
+  #endif
 
+}
 
-void yr_finalize()
+void yr_finalize(void)
 {
   yr_heap_free();
 }
 
+//
+// _yr_set_tidx
+//
+// Set the thread index (tidx) for the current thread. The tidx is the index
+// that will be used by the thread to access thread-specific data stored in
+// YARA_RULES structure.
+//
+// Args:
+//    int tidx   - The zero-based tidx that will be associated to the current
+//                 thread.
+//
+
+void yr_set_tidx(int tidx)
+{
+  #ifdef WIN32
+    //TODO: implement this
+  #else
+    pthread_setspecific(key, (void*) (size_t) (tidx + 1));
+  #endif
+}
 
 
+//
+// _yr_get_tidx
+//
+// Get the thread index (tidx) for the current thread.
+//
+// Returns:
+//    The tidx for the current thread or -1 if the current thread doesn't
+//    have any tidx associated.
+//
 
-
-
-
+int yr_get_tidx(void)
+{
+  return (int) (size_t) pthread_getspecific(key) - 1;
+}
diff --git a/libyara/parser.c b/libyara/parser.c
index b0281b3..6501424 100644
--- a/libyara/parser.c
+++ b/libyara/parser.c
@@ -128,7 +128,7 @@ void yr_parser_emit_pushes_for_strings(
           PTR_TO_UINT64(string),
           NULL);
 
-      string->flags |= STRING_FLAGS_REFERENCED;
+      string->g_flags |= STRING_GFLAGS_REFERENCED;
     }
 
     string = yr_arena_next_address(
@@ -491,9 +491,9 @@ STRING* yr_parser_reduce_string_declaration(
   int min_token_length;
   char* file_name;
   char warning_message[512];
-
   STRING* string;
   YARA_COMPILER* compiler = yyget_extra(yyscanner);
+  int tidx = yr_get_tidx();
 
   compiler->last_result = yr_arena_allocate_struct(
       compiler->strings_arena,
@@ -516,10 +516,10 @@ STRING* yr_parser_reduce_string_declaration(
     return NULL;
 
   if (strcmp(identifier,"$") == 0)
-    flags |= STRING_FLAGS_ANONYMOUS;
+    flags |= STRING_GFLAGS_ANONYMOUS;
 
-  if (!(flags & STRING_FLAGS_WIDE))
-    flags |= STRING_FLAGS_ASCII;
+  if (!(flags & STRING_GFLAGS_WIDE))
+    flags |= STRING_GFLAGS_ASCII;
 
   // The STRING_FLAGS_SINGLE_MATCH flag indicates that finding
   // a single match for the string is enough. This is true in
@@ -527,16 +527,17 @@ STRING* yr_parser_reduce_string_declaration(
   // operators are used. All strings are marked STRING_FLAGS_SINGLE_MATCH
   // initially, and unmarked later if required.
 
-  flags |= STRING_FLAGS_SINGLE_MATCH;
+  flags |= STRING_GFLAGS_SINGLE_MATCH;
 
-  string->flags = flags;
+  string->g_flags = flags;
   string->mask = NULL;
   string->re.regexp = NULL;
   string->re.extra = NULL;
-  string->matches_list_head = NULL;
-  string->matches_list_tail = NULL;
+  string->matches[tidx].head = NULL;
+  string->matches[tidx].tail = NULL;
+
 
-  if (flags & STRING_FLAGS_HEXADECIMAL)
+  if (flags & STRING_GFLAGS_HEXADECIMAL)
   {
     compiler->last_result = yr_parser_new_hex_string(
         compiler,
@@ -547,12 +548,12 @@ STRING* yr_parser_reduce_string_declaration(
   }
   else
   {
-    if (flags & STRING_FLAGS_REGEXP)
+    if (flags & STRING_GFLAGS_REGEXP)
     {
       if (yr_regex_compile(
           &string->re,
           str->c_string,
-          flags & STRING_FLAGS_NO_CASE,
+          flags & STRING_GFLAGS_NO_CASE,
           compiler->last_error_extra_info,
           sizeof(compiler->last_error_extra_info),
           &error_offset) <= 0)
@@ -691,7 +692,7 @@ int yr_parser_reduce_rule_declaration(
   if (compiler->last_result != ERROR_SUCCESS)
     return compiler->last_result;
 
-  rule->flags = flags | compiler->current_rule_flags;
+  rule->g_flags = flags | compiler->current_rule_flags;
   rule->tags = tags;
   rule->strings = strings;
   rule->metas = metas;
@@ -736,7 +737,7 @@ int yr_parser_reduce_string_identifier(
 
         while(!STRING_IS_NULL(string))
         {
-          string->flags &= ~STRING_FLAGS_SINGLE_MATCH;
+          string->g_flags &= ~STRING_GFLAGS_SINGLE_MATCH;
           string = yr_arena_next_address(
               compiler->strings_arena,
               string,
@@ -762,11 +763,11 @@ int yr_parser_reduce_string_identifier(
           NULL);
 
       if (instruction != SFOUND)
-        string->flags &= ~STRING_FLAGS_SINGLE_MATCH;
+        string->g_flags &= ~STRING_GFLAGS_SINGLE_MATCH;
 
       yr_parser_emit(yyscanner, instruction, NULL);
 
-      string->flags |= STRING_FLAGS_REFERENCED;
+      string->g_flags |= STRING_GFLAGS_REFERENCED;
     }
   }
 
diff --git a/libyara/rules.c b/libyara/rules.c
index 699fa84..5c5e379 100644
--- a/libyara/rules.c
+++ b/libyara/rules.c
@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+#include <assert.h>
 #include <string.h>
 #include <time.h>
 
@@ -417,18 +418,19 @@ inline int _yr_scan_verify_string_match(
 
 
 int _yr_scan_verify_match(
-    YARA_RULES* rules,
     AC_MATCH* ac_match,
     uint8_t* data,
     size_t data_size,
-    size_t string_offset)
+    size_t string_offset,
+    ARENA* matches_arena)
 {
   MATCH* match;
   STRING* string;
 
-  int result;
   int32_t match_length;
-
+  int result;
+  int tidx;
+  
   match_length = _yr_scan_verify_string_match(
       ac_match->string,
       data + string_offset,
@@ -438,17 +440,17 @@ int _yr_scan_verify_match(
   if (match_length > 0)
   {
     string = ac_match->string;
-    string->flags |= STRING_FLAGS_FOUND;
+    tidx = yr_get_tidx();
 
-    if (string->matches_list_tail != NULL &&
-        string->matches_list_tail->last_offset == string_offset - 1)
+    if (string->matches[tidx].tail != NULL &&
+        string->matches[tidx].tail->last_offset == string_offset - 1)
     {
-      string->matches_list_tail->last_offset = string_offset;
+      string->matches[tidx].tail->last_offset = string_offset;
     }
     else
     {
       result = yr_arena_allocate_memory(
-          rules->matches_arena,
+          matches_arena,
           sizeof(MATCH),
           (void**) &match);
 
@@ -461,7 +463,7 @@ int _yr_scan_verify_match(
       match->next = NULL;
 
       result = yr_arena_write_data(
-          rules->matches_arena,
+          matches_arena,
           data + string_offset,
           match_length,
           (void**) &match->data);
@@ -469,13 +471,13 @@ int _yr_scan_verify_match(
       if (result != ERROR_SUCCESS)
         return result;
 
-      if (string->matches_list_head == NULL)
-        string->matches_list_head = match;
+      if (string->matches[tidx].head == NULL)
+        string->matches[tidx].head = match;
 
-      if (string->matches_list_tail != NULL)
-        string->matches_list_tail->next = match;
+      if (string->matches[tidx].tail != NULL)
+        string->matches[tidx].tail->next = match;
 
-      string->matches_list_tail = match;
+      string->matches[tidx].tail = match;
     }
   }
 
@@ -483,6 +485,20 @@ int _yr_scan_verify_match(
 }
 
 
+void _yr_rules_lock(
+    YARA_RULES* rules)
+{
+  pthread_mutex_lock(&rules->mutex);
+}
+
+
+void _yr_rules_unlock(
+    YARA_RULES* rules)
+{
+  pthread_mutex_unlock(&rules->mutex);
+}
+
+
 int yr_rules_define_integer_variable(
     YARA_RULES* rules,
     const char* identifier,
@@ -556,34 +572,32 @@ int yr_rules_define_string_variable(
 }
 
 
-void yr_rules_free_matches(
+void _yr_rules_clean_matches(
     YARA_RULES* rules)
 {
   RULE* rule;
   STRING* string;
   MATCH* match;
   MATCH* next_match;
+  
+  int tidx = yr_get_tidx();
 
   rule = rules->rules_list_head;
 
   while (!RULE_IS_NULL(rule))
   {
-    rule->flags &= ~RULE_FLAGS_MATCH;
+    rule->t_flags[tidx] &= ~RULE_TFLAGS_MATCH;
     string = rule->strings;
 
     while (!STRING_IS_NULL(string))
     {
-      string->flags &= ~STRING_FLAGS_FOUND;
-      string->matches_list_head = NULL;
-      string->matches_list_tail = NULL;
+      string->matches[tidx].head = NULL;
+      string->matches[tidx].tail = NULL;
       string++;
     }
 
     rule++;
   }
-
-  if (rules->matches_arena != NULL)
-    yr_arena_destroy(rules->matches_arena);
 }
 
 
@@ -593,7 +607,8 @@ int yr_rules_scan_mem_block(
     size_t data_size,
     int fast_scan_mode,
     int timeout,
-    time_t start_time)
+    time_t start_time,
+    ARENA* matches_arena)
 {
 
   AC_STATE* next_state;
@@ -604,6 +619,7 @@ int yr_rules_scan_mem_block(
   size_t i;
 
   int result;
+  int tidx = yr_get_tidx();
 
   current_state = rules->automaton->root;
   i = 0;
@@ -617,15 +633,15 @@ int yr_rules_scan_mem_block(
       if (i >= ac_match->backtrack)
       {
         if (!(fast_scan_mode &&
-              ac_match->string->flags & STRING_FLAGS_FOUND &&
-              ac_match->string->flags & STRING_FLAGS_SINGLE_MATCH))
+              ac_match->string->matches[tidx].tail != NULL &&
+              STRING_IS_SINGLE_MATCH(ac_match->string)))
         {
           result = _yr_scan_verify_match(
-              rules,
               ac_match,
               data,
               data_size,
-              i - ac_match->backtrack);
+              i - ac_match->backtrack,
+              matches_arena);
 
           if (result != ERROR_SUCCESS)
             return result;
@@ -662,11 +678,11 @@ int yr_rules_scan_mem_block(
   while (ac_match != NULL)
   {
     result = _yr_scan_verify_match(
-        rules,
         ac_match,
         data,
         data_size,
-        data_size - ac_match->backtrack);
+        data_size - ac_match->backtrack,
+        matches_arena);
 
     if (result != ERROR_SUCCESS)
       return result;
@@ -689,22 +705,43 @@ int yr_rules_scan_mem_blocks(
 {
   RULE* rule;
   EVALUATION_CONTEXT context;
+  ARENA* matches_arena = NULL;
 
   time_t start_time;
 
-  char message[512];
-  int result;
+  int message;
+  int tidx;
+  int result = ERROR_SUCCESS;
 
   context.file_size = block->size;
   context.mem_block = block;
   context.entry_point = UNDEFINED;
 
-  yr_rules_free_matches(rules);
+  tidx = yr_get_tidx();
 
-  result = yr_arena_create(&rules->matches_arena);
+  if (tidx == -1) 
+  {
+    _yr_rules_lock(rules);
+
+    tidx = rules->threads_count;
+
+    if (tidx < MAX_THREADS)
+      rules->threads_count++;
+    else
+      result = ERROR_TOO_MANY_THREADS;
+    
+    _yr_rules_unlock(rules);
+
+    if (result != ERROR_SUCCESS)
+      return result;
+
+    yr_set_tidx(tidx);
+  }
+
+  result = yr_arena_create(&matches_arena);
 
   if (result != ERROR_SUCCESS)
-    return result;
+    goto _exit;
 
   start_time = time(NULL);
 
@@ -729,10 +766,11 @@ int yr_rules_scan_mem_blocks(
         block->size,
         fast_scan_mode,
         timeout,
-        start_time);
+        start_time,
+        matches_arena);
 
     if (result != ERROR_SUCCESS)
-      return result;
+      goto _exit;
 
     block = block->next;
   }
@@ -740,16 +778,15 @@ int yr_rules_scan_mem_blocks(
   result = yr_execute_code(rules, &context);
 
   if (result != ERROR_SUCCESS)
-      return result;
+    goto _exit;
 
   rule = rules->rules_list_head;
 
   while (!RULE_IS_NULL(rule))
   {
-    if (rule->flags & RULE_FLAGS_GLOBAL &&
-        !(rule->flags & RULE_FLAGS_MATCH))
+    if (RULE_IS_GLOBAL(rule) && !(rule->t_flags[tidx] & RULE_TFLAGS_MATCH))
     {
-      rule->namespace->flags |= NAMESPACE_FLAGS_UNSATISFIED_GLOBAL;
+      rule->namespace->t_flags[tidx] |= NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL;
     }
 
     rule++;
@@ -759,23 +796,42 @@ int yr_rules_scan_mem_blocks(
 
   while (!RULE_IS_NULL(rule))
   {
-    if (rule->flags & RULE_FLAGS_MATCH &&
-        !(rule->flags & RULE_FLAGS_PRIVATE) &&
-        !(rule->namespace->flags & NAMESPACE_FLAGS_UNSATISFIED_GLOBAL))
+    if (rule->t_flags[tidx] & RULE_TFLAGS_MATCH &&
+        !(rule->namespace->t_flags[tidx] & NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL))
+    {
+      message = CALLBACK_MSG_RULE_MATCHING;
+    }
+    else
     {
-      switch (callback(rule, user_data))
+      message = CALLBACK_MSG_RULE_NOT_MATCHING;
+    }
+
+    if (!RULE_IS_PRIVATE(rule))
+    {
+      switch (callback(message, rule, user_data))
       {
         case CALLBACK_ABORT:
-          return ERROR_SUCCESS;
+          result = ERROR_SUCCESS;
+          goto _exit;
 
         case CALLBACK_ERROR:
-          return ERROR_CALLBACK_ERROR;
+          result = ERROR_CALLBACK_ERROR;
+          goto _exit;
       }
     }
+
     rule++;
   }
 
-  return ERROR_SUCCESS;
+  callback(CALLBACK_MSG_SCAN_FINISHED, NULL, user_data);
+
+_exit:
+  _yr_rules_clean_matches(rules);
+
+  if (matches_arena != NULL)
+    yr_arena_destroy(matches_arena);
+
+  return result;
 }
 
 
@@ -883,6 +939,7 @@ int yr_rules_save(
     YARA_RULES* rules,
     const char* filename)
 {
+  assert(rules->threads_count == 0);
   return yr_arena_save(rules->arena, filename);
 }
 
@@ -917,7 +974,13 @@ int yr_rules_load(
   new_rules->code_start = header->code_start;
   new_rules->externals_list_head = header->externals_list_head;
   new_rules->rules_list_head = header->rules_list_head;
-  new_rules->matches_arena = NULL;
+  new_rules->threads_count = 0;
+
+  #if WIN32
+  new_rules->mutex = CreateMutex(NULL, FALSE, NULL);
+  #else
+  pthread_mutex_init(&new_rules->mutex, NULL);
+  #endif
 
   rule = new_rules->rules_list_head;
 
@@ -964,7 +1027,6 @@ int yr_rules_destroy(
     external++;
   }
 
-  yr_rules_free_matches(rules);
   yr_arena_destroy(rules->arena);
   yr_free(rules);
 
diff --git a/libyara/yara.h b/libyara/yara.h
index 8b17dce..656c413 100644
--- a/libyara/yara.h
+++ b/libyara/yara.h
@@ -22,6 +22,10 @@ limitations under the License.
 
 #ifdef WIN32
 #include <windows.h>
+typedef HANDLE mutex_t;
+#else
+#include <pthread.h>
+typedef pthread_mutex_t mutex_t;
 #endif
 
 #ifdef _MSC_VER
@@ -81,10 +85,22 @@ limitations under the License.
 #define ERROR_TIMEOUT                           35
 #define ERROR_LOOP_NESTING_LIMIT_EXCEEDED       36
 #define ERROR_DUPLICATE_LOOP_IDENTIFIER         37
+#define ERROR_TOO_MANY_THREADS                  38
+
+
+#define CALLBACK_MSG_RULE_MATCHING            1
+#define CALLBACK_MSG_RULE_NOT_MATCHING        2
+#define CALLBACK_MSG_SCAN_FINISHED            3
+
+#define CALLBACK_CONTINUE  0
+#define CALLBACK_ABORT     1
+#define CALLBACK_ERROR     2
+
 
 #define LOOP_LOCAL_VARS 4
 #define MAX_LOOP_NESTING 4
 #define MAX_INCLUDE_DEPTH 16
+#define MAX_THREADS 32
 #define LEX_BUF_SIZE  1024
 
 #ifndef MAX_PATH
@@ -131,53 +147,82 @@ limitations under the License.
 #define EXTERNAL_VARIABLE_IS_NULL(x) \
     ((x) != NULL ? (x)->type == EXTERNAL_VARIABLE_TYPE_NULL : TRUE)
 
-#define CALLBACK_CONTINUE  0
-#define CALLBACK_ABORT     1
-#define CALLBACK_ERROR     2
 
-#define STRING_FLAGS_FOUND          0x01
-#define STRING_FLAGS_REFERENCED     0x02
-#define STRING_FLAGS_HEXADECIMAL    0x04
-#define STRING_FLAGS_NO_CASE        0x08
-#define STRING_FLAGS_ASCII          0x10
-#define STRING_FLAGS_WIDE           0x20
-#define STRING_FLAGS_REGEXP         0x40
-#define STRING_FLAGS_FULL_WORD      0x80
-#define STRING_FLAGS_ANONYMOUS      0x100
-#define STRING_FLAGS_SINGLE_MATCH   0x200
-#define STRING_FLAGS_NULL           0x1000
+#define STRING_TFLAGS_FOUND          0x01
+
+#define STRING_GFLAGS_REFERENCED     0x01
+#define STRING_GFLAGS_HEXADECIMAL    0x02
+#define STRING_GFLAGS_NO_CASE        0x04
+#define STRING_GFLAGS_ASCII          0x08
+#define STRING_GFLAGS_WIDE           0x10
+#define STRING_GFLAGS_REGEXP         0x20
+#define STRING_GFLAGS_FULL_WORD      0x40
+#define STRING_GFLAGS_ANONYMOUS      0x80
+#define STRING_GFLAGS_SINGLE_MATCH   0x100
+#define STRING_GFLAGS_NULL           0x1000
 
 #define STRING_IS_HEX(x) \
-    (((x)->flags) & STRING_FLAGS_HEXADECIMAL)
+    (((x)->g_flags) & STRING_GFLAGS_HEXADECIMAL)
+
 #define STRING_IS_NO_CASE(x) \
-    (((x)->flags) & STRING_FLAGS_NO_CASE)
+    (((x)->g_flags) & STRING_GFLAGS_NO_CASE)
+
 #define STRING_IS_ASCII(x) \
-    (((x)->flags) & STRING_FLAGS_ASCII)
+    (((x)->g_flags) & STRING_GFLAGS_ASCII)
+
 #define STRING_IS_WIDE(x) \
-    (((x)->flags) & STRING_FLAGS_WIDE)
+    (((x)->g_flags) & STRING_GFLAGS_WIDE)
+
 #define STRING_IS_REGEXP(x) \
-    (((x)->flags) & STRING_FLAGS_REGEXP)
+    (((x)->g_flags) & STRING_GFLAGS_REGEXP)
+
 #define STRING_IS_FULL_WORD(x) \
-    (((x)->flags) & STRING_FLAGS_FULL_WORD)
+    (((x)->g_flags) & STRING_GFLAGS_FULL_WORD)
+
 #define STRING_IS_ANONYMOUS(x) \
-    (((x)->flags) & STRING_FLAGS_ANONYMOUS)
+    (((x)->g_flags) & STRING_GFLAGS_ANONYMOUS)
+
 #define STRING_IS_REFERENCED(x) \
-    (((x)->flags) & STRING_FLAGS_REFERENCED)
+    (((x)->g_flags) & STRING_GFLAGS_REFERENCED)
+
+#define STRING_IS_SINGLE_MATCH(x) \
+    (((x)->g_flags) & STRING_GFLAGS_SINGLE_MATCH)
+
 #define STRING_IS_NULL(x) \
-    ((x) == NULL || ((x)->flags) & STRING_FLAGS_NULL)
+    ((x) == NULL || ((x)->g_flags) & STRING_GFLAGS_NULL)
+
+#define STRING_FOUND(x) \
+    ((x)->matches[yr_get_tidx()].tail != NULL)
+
+
+#define RULE_TFLAGS_MATCH                0x01
 
-#define RULE_FLAGS_MATCH                0x01
-#define RULE_FLAGS_PRIVATE              0x02
-#define RULE_FLAGS_GLOBAL               0x04
-#define RULE_FLAGS_REQUIRE_EXECUTABLE   0x08
-#define RULE_FLAGS_REQUIRE_FILE         0x10
-#define RULE_FLAGS_NULL                 0x1000
+#define RULE_GFLAGS_PRIVATE              0x01
+#define RULE_GFLAGS_GLOBAL               0x02
+#define RULE_GFLAGS_REQUIRE_EXECUTABLE   0x04
+#define RULE_GFLAGS_REQUIRE_FILE         0x08
+#define RULE_GFLAGS_NULL                 0x1000
+
+#define RULE_IS_PRIVATE(x) \
+    (((x)->g_flags) & RULE_GFLAGS_PRIVATE)
+
+#define RULE_IS_GLOBAL(x) \
+    (((x)->g_flags) & RULE_GFLAGS_GLOBAL)
 
 #define RULE_IS_NULL(x) \
-    (((x)->flags) & RULE_FLAGS_NULL)
+    (((x)->g_flags) & RULE_GFLAGS_NULL)
+
+#define RULE_MATCHES(x) \
+    ((x)->t_flags[yr_get_tidx()] & RULE_TFLAGS_MATCH)
+
+
+
+#define NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL      0x01
+
+#define NAMESPACE_HAS_UNSATISFIED_GLOBAL(x) \
+    ((x)->t_flags[yr_get_tidx()] & NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL)
 
 
-#define NAMESPACE_FLAGS_UNSATISFIED_GLOBAL      0x01
 
 #define MAX_ARENA_PAGES 32
 
@@ -191,6 +236,8 @@ limitations under the License.
 
 #define PTR_TO_UINT64(x)  ((uint64_t) (size_t) x)
 
+#define STRING_MATCHES(x) (x->matches[yr_get_tidx()])
+
 
 typedef struct _RELOC
 {
@@ -251,7 +298,7 @@ typedef struct _MATCH
 
 typedef struct _NAMESPACE
 {
-  int32_t flags;
+  int32_t t_flags[MAX_THREADS];     // Thread-specific flags
   DECLARE_REFERENCE(char*, name);
 
 } NAMESPACE;
@@ -270,23 +317,27 @@ typedef struct _META
 
 typedef struct _STRING
 {
-  int32_t flags;
+  int32_t g_flags;
   int32_t length;
 
+  REGEXP re;
+
   DECLARE_REFERENCE(char*, identifier);
   DECLARE_REFERENCE(uint8_t*, string);
   DECLARE_REFERENCE(uint8_t*, mask);
-  DECLARE_REFERENCE(MATCH*, matches_list_head);
-  DECLARE_REFERENCE(MATCH*, matches_list_tail);
 
-  REGEXP re;
+  struct {
+    DECLARE_REFERENCE(MATCH*, head);
+    DECLARE_REFERENCE(MATCH*, tail);
+  } matches[MAX_THREADS];
 
 } STRING;
 
 
 typedef struct _RULE
 {
-  int32_t flags;
+  int32_t g_flags;               // Global flags
+  int32_t t_flags[MAX_THREADS];  // Thread-specific flags
 
   DECLARE_REFERENCE(char*, identifier);
   DECLARE_REFERENCE(char*, tags);
@@ -409,6 +460,7 @@ typedef void (*YARAREPORT)(
 
 
 typedef int (*YARACALLBACK)(
+    int message,
     RULE* rule,
     void* data);
 
@@ -478,14 +530,13 @@ typedef struct _MEMORY_BLOCK
 
 typedef struct _YARA_RULES {
 
+  int                  threads_count;
   ARENA*               arena;
-  ARENA*               matches_arena;
   RULE*                rules_list_head;
   EXTERNAL_VARIABLE*   externals_list_head;
   AC_AUTOMATON*        automaton;
   int8_t*              code_start;
-  int                  last_error;
-  char                 last_error_extra_info[256];
+  mutex_t              mutex;
 
 } YARA_RULES;
 
@@ -502,6 +553,9 @@ void yr_initialize(void);
 void yr_finalize(void);
 
 
+int yr_get_tidx(void);
+
+
 int yr_compiler_create(
     YARA_COMPILER** compiler);
 
diff --git a/yara-python/yara-python.c b/yara-python/yara-python.c
index a29611b..6b50f75 100644
--- a/yara-python/yara-python.c
+++ b/yara-python/yara-python.c
@@ -361,7 +361,7 @@ int yara_callback(
   {
     if (string->flags & STRING_FLAGS_FOUND)
     {
-      m = string->matches_list_head;
+      m = STRING_MATCHES(string).head;
 
       while (m != NULL)
       {
diff --git a/yara.c b/yara.c
index 85eac30..027da62 100644
--- a/yara.c
+++ b/yara.c
@@ -1,5 +1,5 @@
 /*
-Copyright (c) 2007. Victor M. Alvarez [plusvic at gmail.com].
+Copyright (c) 2013. Victor M. Alvarez [plusvic at gmail.com].
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,9 +33,31 @@ limitations under the License.
 #include <string.h>
 #include <yara.h>
 
+#include "threading.h"
 #include "config.h"
 #include "REVISION"
 
+#define USAGE \
+"usage:  yara [OPTION]... RULES_FILE FILE | PID\n"\
+"options:\n"\
+"  -t <tag>                 only print rules tagged as <tag>.\n"\
+"  -i <identifier>          only print rules named <identifier>.\n"\
+"  -n                       only print not satisfied rules (negate).\n"\
+"  -g                       print tags.\n"\
+"  -m                       print metadata.\n"\
+"  -s                       print matching strings.\n"\
+"  -l <number>              abort scanning after matching a <number> rules.\n"\
+"  -a <seconds>             abort scanning after a number of seconds has elapsed.\n"\
+"  -d <identifier>=<value>  define external variable.\n"\
+"  -r                       recursively search directories.\n"\
+"  -v                       show version information.\n"
+
+#define EXTERNAL_TYPE_INTEGER   1
+#define EXTERNAL_TYPE_BOOLEAN   2
+#define EXTERNAL_TYPE_STRING    3
+
+#define ERROR_COULD_NOT_CREATE_THREAD  100
+
 #ifndef MAX_PATH
 #define MAX_PATH 255
 #endif
@@ -44,18 +66,7 @@ limitations under the License.
 #define snprintf _snprintf
 #endif
 
-
-int recursive_search = FALSE;
-int show_tags = FALSE;
-int show_specified_tags = FALSE;
-int show_specified_rules = FALSE;
-int show_strings = FALSE;
-int show_meta = FALSE;
-int fast_scan = FALSE;
-int negate = FALSE;
-int count = 0;
-int limit = 0;
-int timeout = 0;
+#define MAX_QUEUED_FILES 64
 
 
 typedef struct _TAG
@@ -74,10 +85,6 @@ typedef struct _IDENTIFIER
 } IDENTIFIER;
 
 
-#define EXTERNAL_TYPE_INTEGER   1
-#define EXTERNAL_TYPE_BOOLEAN   2
-#define EXTERNAL_TYPE_STRING    3
-
 typedef struct _EXTERNAL
 {
   char type;
@@ -92,42 +99,118 @@ typedef struct _EXTERNAL
 } EXTERNAL;
 
 
+typedef struct _QUEUED_FILE {
+
+  char* path;
+  ARENA* output;
+
+} QUEUED_FILE;
+
+
+int recursive_search = FALSE;
+int show_tags = FALSE;
+int show_specified_tags = FALSE;
+int show_specified_rules = FALSE;
+int show_strings = FALSE;
+int show_meta = FALSE;
+int fast_scan = FALSE;
+int negate = FALSE;
+int count = 0;
+int limit = 0;
+int timeout = 0;
+int threads = 8;
+
+
 TAG* specified_tags_list = NULL;
 IDENTIFIER* specified_rules_list = NULL;
 EXTERNAL* externals_list = NULL;
 
-#define USAGE \
-"usage:  yara [OPTION]... RULES_FILE FILE | PID\n"\
-"options:\n"\
-"  -t <tag>                 only print rules tagged as <tag>.\n"\
-"  -i <identifier>          only print rules named <identifier>.\n"\
-"  -n                       only print not satisfied rules (negate).\n"\
-"  -g                       print tags.\n"\
-"  -m                       print metadata.\n"\
-"  -s                       print matching strings.\n"\
-"  -l <number>              abort scanning after matching a <number> rules.\n"\
-"  -a <seconds>             abort scanning after a number of seconds has elapsed.\n"\
-"  -d <identifier>=<value>  define external variable.\n"\
-"  -r                       recursively search directories.\n"\
-"  -v                       show version information.\n"
 
-void show_help()
+// file_queue is size-limited queue stored as a circular array, files are
+// removed from queue_head position and new files are added at queue_tail
+// position. The array has room for one extra element to avoid queue_head
+// being equal to queue_tail in a full queue. The only situation where 
+// queue_head == queue_tail is when queue is empty.
+
+QUEUED_FILE file_queue[MAX_QUEUED_FILES + 1];
+
+int queue_head;
+int queue_tail;
+
+SEMAPHORE used_slots;
+SEMAPHORE unused_slots;
+
+MUTEX queue_mutex;
+MUTEX output_mutex;
+
+
+void file_queue_init()
 {
-  printf(USAGE);
-  printf("\nReport bugs to: <%s>\n", PACKAGE_BUGREPORT);
+  queue_tail = 0;
+  queue_head = 0;
+
+  mutex_init(&queue_mutex);
+  semaphore_init(&used_slots, 0);
+  semaphore_init(&unused_slots, MAX_QUEUED_FILES);
 }
 
 
-int is_numeric(
-    const char *str)
+void file_queue_destroy()
 {
-  while(*str)
+  mutex_destroy(&queue_mutex);
+  semaphore_destroy(&unused_slots);
+  semaphore_destroy(&used_slots);
+}
+
+
+void file_queue_finish()
+{
+  int i;
+
+  for (i = 0; i < MAX_THREADS; i++)
+    semaphore_release(&used_slots);
+}
+
+
+void file_queue_put(
+    const char* file_path)
+{
+  semaphore_wait(&unused_slots);
+  mutex_lock(&queue_mutex);
+
+  file_queue[queue_tail].path = strdup(file_path);
+
+  //TODO: handle errors
+  yr_arena_create(&file_queue[queue_tail].output);
+
+  queue_tail = (queue_tail + 1) % (MAX_QUEUED_FILES + 1);
+
+  mutex_unlock(&queue_mutex);
+  semaphore_release(&used_slots);
+}
+
+
+char* file_queue_get()
+{
+  char* result;
+
+  semaphore_wait(&used_slots);
+  mutex_lock(&queue_mutex);
+
+  if (queue_head == queue_tail) // queue is empty
   {
-    if(!isdigit(*str++))
-      return 0;
+    result = NULL;
+  }
+  else
+  {
+    result = file_queue[queue_head].path;
+    queue_head = (queue_head + 1) % (MAX_QUEUED_FILES + 1);
   }
 
-  return 1;
+  mutex_unlock(&queue_mutex);
+  semaphore_release(&unused_slots);
+
+  return result;
 }
 
 
@@ -137,16 +220,12 @@ int is_directory(
     const char* path)
 {
   if (GetFileAttributes(path) & FILE_ATTRIBUTE_DIRECTORY)
-  {
     return TRUE;
-  }
   else
-  {
     return FALSE;
-  }
 }
 
-int scan_dir(
+void scan_dir(
     const char* dir,
     int recursive,
     YARA_RULES* rules,
@@ -158,8 +237,6 @@ int scan_dir(
   char full_path[MAX_PATH];
   static char path_and_mask[MAX_PATH];
 
-  int result = ERROR_SUCCESS;
-
   snprintf(path_and_mask, sizeof(path_and_mask), "%s\\*", dir);
 
   hFind = FindFirstFile(path_and_mask, &FindFileData);
@@ -173,28 +250,17 @@ int scan_dir(
 
       if (!(FindFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY))
       {
-        result = yr_rules_scan_file(
-            rules,
-            full_path,
-            callback,
-            full_path,
-            fast_scan,
-            timeout);
+        file_queue_put(full_path);
       }
       else if (recursive && FindFileData.cFileName[0] != '.' )
       {
-        result = scan_dir(full_path, recursive, rules, callback);
+        scan_dir(full_path, recursive, rules, callback);
       }
 
-      if (result != ERROR_SUCCESS)
-        break;
-
     } while (FindNextFile(hFind, &FindFileData));
 
     FindClose(hFind);
   }
-
-  return result;
 }
 
 #else
@@ -205,14 +271,12 @@ int is_directory(
   struct stat st;
 
   if (stat(path,&st) == 0)
-  {
     return S_ISDIR(st.st_mode);
-  }
 
   return 0;
 }
 
-int scan_dir(
+void scan_dir(
     const char* dir,
     int recursive,
     YARA_RULES* rules,
@@ -223,8 +287,6 @@ int scan_dir(
   struct stat st;
   char full_path[MAX_PATH];
 
-  int result = ERROR_SUCCESS;
-
   dp = opendir(dir);
 
   if (dp)
@@ -235,27 +297,21 @@ int scan_dir(
     {
       snprintf(full_path, sizeof(full_path), "%s/%s", dir, de->d_name);
 
-      int err = stat(full_path,&st);
+      int err = lstat(full_path, &st);
 
       if (err == 0)
       {
         if(S_ISREG(st.st_mode))
         {
-          result = yr_rules_scan_file(
-              rules,
-              full_path,
-              callback,
-              full_path,
-              fast_scan,
-              timeout);
+          file_queue_put(full_path);
         }
-        else if(recursive && S_ISDIR(st.st_mode) && de->d_name[0] != '.')
+        else if(recursive &&  
+                S_ISDIR(st.st_mode) &&
+                !S_ISLNK(st.st_mode) &&
+                de->d_name[0] != '.')
         {
-          result = scan_dir(full_path, recursive, rules, callback);
+          scan_dir(full_path, recursive, rules, callback);
         }
-
-        if (result != ERROR_SUCCESS)
-          break;
       }
 
       de = readdir(dp);
@@ -263,8 +319,6 @@ int scan_dir(
 
     closedir(dp);
   }
-
-  return result;
 }
 
 #endif
@@ -282,13 +336,9 @@ void print_string(
   for (i = 0; i < length; i++)
   {
     if (str[i] >= 32 && str[i] <= 126)
-    {
       printf("%c",str[i]);
-    }
     else
-    {
       printf("\\x%02x", str[i]);
-    }
 
     if (unicode) i++;
   }
@@ -303,15 +353,54 @@ void print_hex_string(
   unsigned int i;
 
   for (i = 0; i < length; i++)
-  {
     printf("%02X ", data[i]);
-  }
 
   printf("\n");
 }
 
 
-int callback(RULE* rule, void* data)
+void print_scanning_error(int error)
+{
+  switch (error)
+  {
+    case ERROR_SUCCESS:
+      break;
+    case ERROR_COULD_NOT_ATTACH_TO_PROCESS:
+      fprintf(stderr, "can not attach to process (try running as root)\n");
+      break;
+    case ERROR_INSUFICIENT_MEMORY:
+      fprintf(stderr, "not enough memory\n");
+      break;
+    case ERROR_TIMEOUT:
+      fprintf(stderr, "scanning timed out\n");
+      break;
+    case ERROR_COULD_NOT_OPEN_FILE:
+      fprintf(stderr, "could not open file\n");
+      break;
+    case ERROR_ZERO_LENGTH_FILE:
+      fprintf(stderr, "zero length file\n");
+      break;
+    default:
+      fprintf(stderr, "internal error: %d\n", error);
+      break;
+  }
+}
+
+
+void print_compiler_error(
+    int error_level,
+    const char* file_name,
+    int line_number,
+    const char* message)
+{
+  if (error_level == YARA_ERROR_LEVEL_ERROR)
+    fprintf(stderr, "%s(%d): error: %s\n", file_name, line_number, message);
+  else
+    fprintf(stderr, "%s(%d): warning: %s\n", file_name, line_number, message);
+}
+
+
+int handle_message(int message, RULE* rule, void* data)
 {
   TAG* tag;
   IDENTIFIER* identifier;
@@ -321,7 +410,7 @@ int callback(RULE* rule, void* data)
 
   char* tag_name;
   size_t tag_length;
-  int rule_match;
+  int is_matching;
   int string_found;
   int show = TRUE;
 
@@ -368,12 +457,13 @@ int callback(RULE* rule, void* data)
     }
   }
 
-  rule_match = (rule->flags & RULE_FLAGS_MATCH);
+  is_matching = (message == CALLBACK_MSG_RULE_MATCHING);
 
-  show = show && ((!negate && rule_match) || (negate && !rule_match));
+  show = show && ((!negate && is_matching) || (negate && !is_matching));
 
   if (show)
   {
+    mutex_lock(&output_mutex);
     printf("%s ", rule->identifier);
 
     if (show_tags)
@@ -432,11 +522,11 @@ int callback(RULE* rule, void* data)
 
       while (!STRING_IS_NULL(string))
       {
-        string_found = string->flags & STRING_FLAGS_FOUND;
+        string_found = STRING_FOUND(string);
 
         if (string_found)
         {
-          match = string->matches_list_head;
+          match = STRING_MATCHES(string).head;
 
           while (match != NULL)
           {
@@ -462,9 +552,11 @@ int callback(RULE* rule, void* data)
         string++;
       }
     }
+
+    mutex_unlock(&output_mutex);
   }
 
-  if (rule_match)
+  if (is_matching)
     count++;
 
   if (limit != 0 && count >= limit)
@@ -474,6 +566,103 @@ int callback(RULE* rule, void* data)
 }
 
 
+int callback(int message, RULE* rule, void* data)
+{
+  switch(message)
+  {
+    case CALLBACK_MSG_RULE_MATCHING:
+    case CALLBACK_MSG_RULE_NOT_MATCHING:
+      return handle_message(message, rule, data);
+  }
+}
+
+#ifdef WIN32
+DWORD WINAPI ThreadProc(LPVOID param)
+#else
+void* scanning_thread(void* param)
+#endif
+{
+  YARA_RULES* rules = (YARA_RULES*) param;
+  char* file_path;
+  int result;
+
+  file_path = file_queue_get();
+
+  while (file_path != NULL) 
+  {
+    result = yr_rules_scan_file(
+        rules,
+        file_path,
+        callback,
+        file_path,
+        fast_scan,
+        timeout);
+
+    if (result != ERROR_SUCCESS)
+    {
+      mutex_lock(&output_mutex);
+      fprintf(stderr, "Error scanning %s: ", file_path);
+      print_scanning_error(result);
+      mutex_unlock(&output_mutex);
+    }
+
+    free(file_path);
+    file_path = file_queue_get();
+  }
+}
+
+
+void cleanup()
+{
+  IDENTIFIER* identifier;
+  IDENTIFIER* next_identifier;
+  TAG* tag;
+  TAG* next_tag;
+  EXTERNAL* external;
+  EXTERNAL* next_external;
+
+  tag = specified_tags_list;
+
+  while(tag != NULL)
+  {
+    next_tag = tag->next;
+    free(tag);
+    tag = next_tag;
+  }
+
+  external = externals_list;
+
+  while(external != NULL)
+  {
+    next_external = external->next;
+    free(external);
+    external = next_external;
+  }
+
+  identifier = specified_rules_list;
+
+  while(identifier != NULL)
+  {
+    next_identifier = identifier->next;
+    free(identifier);
+    identifier = next_identifier;
+  }
+}
+
+
+int is_numeric(
+    const char *str)
+{
+  while(*str)
+  {
+    if(!isdigit(*str++))
+      return 0;
+  }
+
+  return 1;
+}
+
+
 int process_cmd_line(
     int argc,
     char const* argv[])
@@ -602,12 +791,11 @@ int process_cmd_line(
         break;
 
       case '?':
-
         if (optopt == 't')
         {
           fprintf(stderr, "Option -%c requires an argument.\n", optopt);
         }
-        else if (isprint (optopt))
+        else if (isprint(optopt))
         {
           fprintf(stderr, "Unknown option `-%c'.\n", optopt);
         }
@@ -626,54 +814,11 @@ int process_cmd_line(
 
 }
 
-void report_error(
-    int error_level,
-    const char* file_name,
-    int line_number,
-    const char* message)
-{
-  if (error_level == YARA_ERROR_LEVEL_ERROR)
-    fprintf(stderr, "%s(%d): error: %s\n", file_name, line_number, message);
-  else
-    fprintf(stderr, "%s(%d): warning: %s\n", file_name, line_number, message);
-}
 
-
-void cleanup()
+void show_help()
 {
-  IDENTIFIER* identifier;
-  IDENTIFIER* next_identifier;
-  TAG* tag;
-  TAG* next_tag;
-  EXTERNAL* external;
-  EXTERNAL* next_external;
-
-  tag = specified_tags_list;
-
-  while(tag != NULL)
-  {
-    next_tag = tag->next;
-    free(tag);
-    tag = next_tag;
-  }
-
-  external = externals_list;
-
-  while(external != NULL)
-  {
-    next_external = external->next;
-    free(external);
-    external = next_external;
-  }
-
-  identifier = specified_rules_list;
-
-  while(identifier != NULL)
-  {
-    next_identifier = identifier->next;
-    free(identifier);
-    identifier = next_identifier;
-  }
+  printf(USAGE);
+  printf("\nReport bugs to: <%s>\n", PACKAGE_BUGREPORT);
 }
 
 
@@ -687,9 +832,11 @@ int main(
   EXTERNAL* external;
 
   int pid;
+  int i;
   int errors;
   int result;
 
+  THREAD thread[MAX_THREADS];
   clock_t start, end;
 
   if (!process_cmd_line(argc, argv))
@@ -770,7 +917,7 @@ int main(
       external = external->next;
     }
 
-    compiler->error_report_function = report_error;
+    compiler->error_report_function = print_compiler_error;
     rule_file = fopen(argv[optind], "r");
 
     if (rule_file != NULL)
@@ -799,6 +946,8 @@ int main(
     }
   }
 
+  mutex_init(&output_mutex);
+
   if (is_numeric(argv[argc - 1]))
   {
     pid = atoi(argv[argc - 1]);
@@ -809,14 +958,33 @@ int main(
         (void*) argv[argc - 1],
         fast_scan,
         timeout);
+
+    if (result != ERROR_SUCCESS)
+      print_scanning_error(result);
   }
   else if (is_directory(argv[argc - 1]))
-  {
-    result = scan_dir(
+  {   
+    file_queue_init();
+    
+    for (i = 0; i < threads; i++)
+    {
+      if (create_thread(&thread[i], scanning_thread, (void*) rules) != 0)
+        return ERROR_COULD_NOT_CREATE_THREAD;
+    }
+
+    scan_dir(
         argv[argc - 1],
         recursive_search,
         rules,
         callback);
+
+    file_queue_finish();
+
+    // Wait for scan threads to finish
+    for (i = 0; i < threads; i++)
+      thread_join(&thread[i]);
+
+    file_queue_destroy();
   }
   else
   {
@@ -832,32 +1000,21 @@ int main(
 
     end = clock();
 
-    printf( "Scanning time: %f s\n", (float)(end - start) / CLOCKS_PER_SEC);
-  }
-
-  switch (result)
-  {
-    case ERROR_SUCCESS:
-      break;
-    case ERROR_COULD_NOT_ATTACH_TO_PROCESS:
-      fprintf(stderr, "can not attach to process (try running as root)\n");
-      break;
-    case ERROR_INSUFICIENT_MEMORY:
-      fprintf(stderr, "not enough memory\n");
-      break;
-    case ERROR_TIMEOUT:
-      fprintf(stderr, "scanning timed out\n");
-      break;
-    case ERROR_COULD_NOT_OPEN_FILE:
-      fprintf(stderr, "could not open file\n");
-      break;
-    default:
-      fprintf(stderr, "internal error: %d\n", result);
-      break;
+    if (result != ERROR_SUCCESS)
+    {
+      fprintf(stderr, "Error scanning %s: ", argv[argc - 1]);
+      print_scanning_error(result);
+    }
+    else
+    {
+      printf( "Scanning time: %f s\n", (float)(end - start) / CLOCKS_PER_SEC);
+    }
   }
 
   yr_rules_destroy(rules);
   yr_finalize();
+
+  mutex_destroy(&output_mutex);
   cleanup();
 
   return 1;

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/forensics/yara.git



More information about the forensics-changes mailing list