Refactor BLAST::Report to lazily generate reports

In some cases (e.g. when used by some extensions) it is beneficial
to not generate the report files upfront, and only do it when it's
required. Performance gains are significant with large result sets
(i.e. hundreds of MB or GBs) as the data does not need to be
loaded into the process memory.

Lazy method evaluation with memoization allows to achieve just that
without cognitive overload to developer - operations are now only
executed when the methods are invoked, not up-front by default.

Implementation logic was not changed and regression test prepared
upfront is not failing and producing identical results.

Author: tadast

lib/sequenceserver/blast/report.rb

@@ -24,20 +24,11 @@ module BLAST
     class Report < Report
       def initialize(job)
         super do
-          @queries = []
           @querydb = job.databases
         end
       end
 
-      # Attributes parsed out from BLAST output.
-      attr_reader :program, :program_version, :stats, :queries
-
-      # Attributes parsed from job metadata and BLAST output.
-      attr_reader :querydb, :dbtype, :params
-
       def to_json(*_args)
-        generate
-
         %i[querydb program program_version params stats
            queries].inject({}) do |h, k|
           h[k] = send(k)
@@ -53,83 +44,89 @@ def to_json(*_args)
       def xml_file_size
         return File.size(job.imported_xml_file) if job.imported_xml_file
 
-        generate
-
         xml_formatter.size
       end
 
-      # Generate report.
-      def generate
-        return self if @_generated
-
-        job.raise!
-        xml_ir = nil
-        tsv_ir = nil
-        if job.imported_xml_file
-          xml_ir = parse_xml File.read(job.imported_xml_file)
-          tsv_ir = Hash.new do |h1, k1|
-            h1[k1] = Hash.new do |h2, k2|
-              h2[k2] = ['', '', []]
-            end
-          end
-        else
-          xml_ir = parse_xml(xml_formatter.read_file)
-          tsv_ir = parse_tsv(tsv_formatter.read_file)
+      def done?
+        return true if job.imported_xml_file
+
+        File.exist?(xml_formatter.filepath) && File.exist?(tsv_formatter.filepath)
+      end
+
+      def program
+        @program ||= xml_ir[0]
+      end
+
+      def program_version
+        @program_version ||= xml_ir[1]
+      end
+
+      def querydb
+        @querydb ||= xml_ir[3].split.map do |path|
+          { title: File.basename(path) }
         end
-        extract_program_info xml_ir
-        extract_db_info xml_ir
-        extract_params xml_ir
-        extract_stats xml_ir
-        extract_queries xml_ir, tsv_ir
+      end
+
+      def dbtype
+        @dbtype ||= querydb&.first&.type || dbtype_from_program
+      end
 
-        @_generated = true
+      def params
+        @params ||= extract_params
+      end
 
-        self
+      def stats
+        @stats ||= extract_stats
       end
 
-      def done?
-        return true if job.imported_xml_file
+      def queries
+        @queries ||= xml_ir[8].map do |n|
+          query = Query.new(self, n[0], n[2], n[3], [])
+          query.hits = query_hits(n[4], tsv_ir[query.id], query)
 
-        File.exist?(xml_formatter.filepath) && File.exist?(tsv_formatter.filepath)
+          query
+        end
       end
 
       private
 
-      def xml_formatter
-        @xml_formatter ||= Formatter.run(job, 'xml')
+      def xml_ir
+        @xml_ir ||=
+          if job.imported_xml_file
+            parse_xml File.read(job.imported_xml_file)
+          else
+            job.raise!
+            parse_xml(xml_formatter.read_file)
+          end
       end
 
-      def tsv_formatter
-        @tsv_formatter ||= Formatter.run(job, 'custom_tsv')
+      def tsv_ir
+        @tsv_ir ||=
+          if job.imported_xml_file
+            Hash.new do |h1, k1|
+              h1[k1] = Hash.new do |h2, k2|
+                h2[k2] = ['', '', []]
+              end
+            end
+          else
+            job.raise!
+            parse_tsv(tsv_formatter.read_file)
+          end
       end
 
-      # Make program name and program name + version available via `program`
-      # and `program_version` attributes.
-      def extract_program_info(ir)
-        @program         = ir[0]
-        @program_version = ir[1]
+      def xml_formatter
+        @xml_formatter ||= Formatter.run(job, 'xml')
       end
 
-      # Get database information (title and type) from job yaml or from XML.
-      # Sets `querydb` and `dbtype` attributes.
-      def extract_db_info(ir)
-        if @querydb.empty?
-          @querydb = ir[3].split.map do |path|
-            { title: File.basename(path) }
-          end
-          @dbtype = dbtype_from_program
-        else
-          @dbtype = @querydb.first.type
-        end
+      def tsv_formatter
+        @tsv_formatter ||= Formatter.run(job, 'custom_tsv')
       end
 
-      # Make search params available via `params` attribute.
-      #
       # Search params tweak the results. Like evalue cutoff or penalty to open
       # a gap. BLAST+ doesn't list all input params in the XML output. Only
       # matrix, evalue, gapopen, gapextend, and filters are available from XML
       # output.
-      def extract_params(ir)
+      def extract_params
         # Parse/get params from the job first.
         job_params = parse_advanced(job.advanced)
         # Old jobs from beta releases may not have the advanced key but they
@@ -138,21 +135,19 @@ def extract_params(ir)
 
         # Parse params from BLAST XML.
         @params = Hash[
-          *ir[7].first.map { |k, v| [k.gsub('Parameters_', ''), v] }.flatten
+          *xml_ir[7].first.map { |k, v| [k.gsub('Parameters_', ''), v] }.flatten
         ]
         @params['evalue'] = @params.delete('expect')
 
         # Merge into job_params.
         @params = job_params.merge(@params)
       end
 
-      # Make search stats available via `stats` attribute.
-      #
       # Search stats are computed metrics. Like total number of sequences or
       # effective search space.
-      def extract_stats(ir)
-        stats  = ir[8].first[5][0]
-        @stats = {
+      def extract_stats
+        stats = xml_ir[8].first[5][0]
+        {
           nsequences: stats[0],
           ncharacters: stats[1],
           hsp_length: stats[2],
@@ -163,20 +158,11 @@ def extract_stats(ir)
         }
       end
 
-      # Create query objects for the given report from the given ir.
-      def extract_queries(xml_ir, tsv_ir)
-        xml_ir[8].each do |n|
-          query = Query.new(self, n[0], n[2], n[3], [])
-          extract_hits(n[4], tsv_ir[query.id], query)
-          queries << query
-        end
-      end
-
       # Create Hit objects for the given query from the given ir.
-      def extract_hits(xml_ir, tsv_ir, query)
-        return if xml_ir == ["\n"] # => No hits.
+      def query_hits(xml_ir, tsv_ir, query)
+        return [] if xml_ir == ["\n"] # => No hits.
 
-        xml_ir.each do |n|
+        xml_ir.map do |n|
           # If hit comes from a non -parse_seqids database, then id (n[1]) is a
           # BLAST assigned internal id of the format 'gnl|BL_ORD_ID|serial'. We
           # assign the id to accession (because we use accession for sequence
@@ -190,19 +176,21 @@ def extract_hits(xml_ir, tsv_ir, query)
             n[1] = defline.shift
             n[2] = defline.join(' ')
           end
+
           hit = Hit.new(query, n[0], n[1], n[3], n[2], n[4],
                         tsv_ir[n[1]][0], tsv_ir[n[1]][1], [])
-          extract_hsps(n[5], tsv_ir[n[1]][2], hit)
-          query.hits << hit
+
+          hit.hsps = hsps(n[5], tsv_ir[n[1]][2], hit)
+
+          hit
         end
       end
 
-      # Create HSP objects for the given hit from the given ir.
-      def extract_hsps(xml_ir, tsv_ir, hit)
-        xml_ir.each_with_index do |n, i|
+      def hsps(xml_ir, tsv_ir, hit)
+        xml_ir.map.with_index do |n, i|
           n.insert(14, tsv_ir[i])
-          hsp = HSP.new(hit, *n)
-          hit.hsps << hsp
+
+          HSP.new(hit, *n)
         end
       end
 

lib/sequenceserver/report.rb

@@ -8,7 +8,7 @@ module SequenceServer
   class Report
     class << self
       def generate(job)
-        BLAST::Report.new(job).generate
+        BLAST::Report.new(job)
       end
     end