Annotate some more tests that typically would not be encoded, but could be decoded.

  - `README.md`: Clarify since the spec does not determine behavior when something is invalid IAMF that a production system may still try to find a robust way to handle "should-fail" cases.
  - `test_000017`: Document the encoder may have a bug where it is too strict with `num_samples_to_trim_at_end`.
  - `test_000026`: Encoder refuses to produce it because "Output Channel Count SHALL be set to 2", but the decoder should process it anyway because "Output Channel Count can be ignored ".
  - `test_000119`: Encoder refuses to produce it because the codec ID is not known, but the decoder can ignore the mix related to that codec.
  - `test_000502`: Encoder refuses to produce it because `num_submixes` "SHALL NOT be set to 0", but the decoder can process the other mix.
  - Files based on AOMediaCodec/iamf-tools@468eb9b4

Author: jwcullen

tests/README.md

@@ -20,9 +20,11 @@ Theses file describe metadata about the test vector to encode an
     would be understood by a compliant decoder. False when one or more mixes
     exercise fields or features which would cause mixes to be ignored.
 -   `is_valid_to_decode`: True when an IAMF-compliant decoder could decode at
-    least least one mix of the associated IA Sequence ("should-pass"). False
-    when all mixes are non-conformant and would fail to be decoded
-    ("should-false").
+    least one mix of the associated IA Sequence ("should-pass"). False when all
+    mixes are non-conformant and may fail to be decoded ("should-fail"). The
+    IAMF spec does not specify what happens when requirements are violated; a
+    robust system may still attempt to process and create output for
+    "should-fail" tests.
 -   `human_readable_descriptions`: A short description of what is being tested
     and why.
 -   `mp4_fixed_timestamp`: The timestamp within the MP4 file. Can be safely

tests/test_000017.textproto

@@ -13,11 +13,12 @@
 
 test_vector_metadata {
   human_readable_description:
-    "A stereo IAMF stream. It is invalid because it has one frame of samples "
-    "fully trimmed off the end."
+    "A stereo IAMF stream. It has one frame of samples fully trimmed off the "
+    "end."
   file_name_prefix: "test_000017"
+  # TODO(b/356905554): Allow encoding fully trimmed frames.
   is_valid: false
-  is_valid_to_decode: false
+  is_valid_to_decode: true
   validate_user_loudness: true
   mp4_fixed_timestamp: "2023-04-19 00:00:00"
   primary_tested_spec_sections: [

tests/test_000026.textproto

@@ -16,8 +16,11 @@ test_vector_metadata {
     "A stereo IAMF stream encoded using Opus, with an incorrect "
     "`output_channel_count`."
   file_name_prefix: "test_000026"
+  # The encoder would not normally encode data with invalid
+  # `output_channel_count`.
   is_valid: false
-  is_valid_to_decode: false
+  # A compliant decoder should ignore `output_channel_count`.
+  is_valid_to_decode: true
   validate_user_loudness: true
   mp4_fixed_timestamp: "2023-05-05 00:00:00"
   primary_tested_spec_sections: [

tests/test_000119.textproto

@@ -18,8 +18,10 @@ test_vector_metadata {
     "element encoded with an imaginary `codec_id`. For convenience, the "
     "imaginary `codec_id` uses the same syntax as LPCM."
   file_name_prefix: "test_000119"
+  # The encoder cannot accurately encode a `coded_id` which is not yet defined.
   is_valid: false
-  is_valid_to_decode: false
+  # A compliant decoder can decode the mix with `mix_presentation_id` 42.
+  is_valid_to_decode: true
   partition_mix_gain_parameter_blocks: false
   validate_user_loudness: true
   mp4_fixed_timestamp: "2024-07-17 00:00:00"

tests/test_000502.textproto

@@ -14,10 +14,13 @@
 test_vector_metadata {
   human_readable_description:
     "An invalid IAMF stream with two Mix Presentation OBUs. The stream is "
-    "invalid because one of the Mix Presentation OBUs has 0 sub-mixes."
+    "invalid because one of the Mix Presentation OBUs has zero sub-mixes."
   file_name_prefix: "test_000502"
+  # The encoder should not normally encode a mix presentation with zero
+  # sub-mixes.
   is_valid: false
-  is_valid_to_decode: false
+  # A compliant decoder can decode the mix with `mix_presentation_id` 42.
+  is_valid_to_decode: true
   validate_user_loudness: true
   mp4_fixed_timestamp: "2023-08-07 00:00:00"
   primary_tested_spec_sections: [

tests/test_summary.csv

@@ -10,7 +10,7 @@
 "test_000014","test_000005","3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/pre_skip",False,False,"A stereo IAMF stream with no trimming encoded using Opus. It is invalid because `pre_skip` is incorrectly set to 0."
 "test_000015","test_000005","3.8/parameter_id",True,True,"A stereo IAMF stream that includes an extra parameter block which is not used by any audio element or mix presentation."
 "test_000016","test_000005","3.5/num_samples_per_frame, 3.6.1/param_definition_mode, 3.8/duration, 3.8/constant_subblock_duration, 3.9/Audio Frame OBU Syntax and Semantics",False,False,"A stereo IAMF stream. It has a parameter block not aligned with the audio frames, ending 100 samples earlier. This is invalid because parameter blocks should cover the whole duration of the audio frames that use the parameter."
-"test_000017","test_000005","3.2/num_samples_to_trim_at_end, 3.5/num_samples_per_frame",False,False,"A stereo IAMF stream. It is invalid because it has one frame of samples fully trimmed off the end."
+"test_000017","test_000005","3.2/num_samples_to_trim_at_end, 3.5/num_samples_per_frame",False,True,"A stereo IAMF stream. It has one frame of samples fully trimmed off the end."
 "test_000018","test_000005","3.1/obu_type, 3.9/explicit_audio_substream_id",True,True,"A stereo IAMF stream. The substream has an ID larger than 21 which requires the ID to be explicitly listed in the OBU."
 "test_000019","test_000005","3.8/Parameter Block OBU Syntax and Semantics",True,True,"A stereo IAMF stream that has two parameter blocks with the same ID occuring one after another."
 "test_000020","test_000014","3.5/audio_roll_distance, 3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/OPUS Specific, 3.11.1/num_samples_per_frame, 6.2.2/roll",True,True,"A stereo IAMF stream encoded using 20 ms Opus frames which encodes a 500 ms sine wav with a frequency of 1000 Hz."
@@ -19,7 +19,7 @@
 "test_000023","test_000020","3.2/num_samples_to_trim_at_start, 3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/OPUS Specific, 3.11.1/num_samples_per_frame",True,True,"A stereo IAMF stream encoded using 5 ms Opus frames which requires fully trimmed frames at the start."
 "test_000024","test_000020","3.5/audio_roll_distance, 3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/OPUS Specific, 3.11.1/num_samples_per_frame",True,True,"A stereo IAMF stream encoded using 60 ms Opus frames."
 "test_000025","test_000020","3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/version",False,False,"A stereo IAMF stream encoded using Opus, with an incorrect Opus version."
-"test_000026","test_000020","3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/output_channel_count",False,False,"A stereo IAMF stream encoded using Opus, with an incorrect `output_channel_count`."
+"test_000026","test_000020","3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/output_channel_count",False,True,"A stereo IAMF stream encoded using Opus, with an incorrect `output_channel_count`."
 "test_000027","test_000020","3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/output_gain",False,False,"A stereo IAMF stream encoded using Opus, with an incorrect `output_gain`."
 "test_000028","test_000020","3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/channel_mapping_family",False,False,"A stereo IAMF stream encoded using Opus, with an incorrect `channel_mapping_family`."
 "test_000029","test_000005","3.11.4/sample_rate",True,True,"A stereo IAMF stream encoded using LPCM with a sample rate of 48 kHz."
@@ -108,7 +108,7 @@
 "test_000116","test_000020","3.2/obu_type, 3.5/audio_roll_distance, 3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/OPUS Specific, 3.11.1/num_samples_per_frame, 4/Reserved Descriptor OBU, 4/Reserved Temporal Unit OBU, 6.2.2/roll",True,True,"A stereo Opus IAMF stream which additionally contains several reserved OBUs interlaced between the descriptor OBUs and in the first temporal unit. The spec does not predetermine which reserved OBUs will be descriptor OBUs and which will be temporal unit OBUs."
 "test_000117","test_000020","3.2/obu_extension_flag, 3.5/audio_roll_distance, 3.9/Audio Frame OBU Syntax and Semantics, 3.11.1/OPUS Specific, 3.11.1/num_samples_per_frame, 6.2.2/roll",True,True,"A stereo Opus IAMF stream which encodes a 500 ms sine wav with a frequency of 1000 Hz, and where the Audio Element OBU has extension header bytes."
 "test_000118","test_000087","3.6.2/loudspeaker_layout == Stereo, 3.6.2/loudspeaker_layout == 5.1, 3.7/num_audio_elements, 3.7.4/default_mix_gain, 3.11.1/OPUS Specific, 4.2/Simple Profile, 4.2/Base Profile, 8.5.1/Loudness Information",True,True,"A two-layer 5.1 + stereo IAMF stream encoded using Opus and with two Mix Presentations. The first Mix Presentation uses both Audio Elements. The second Mix Presentation uses only the stereo Audio Element. The primary profile is set to Simple and the additional profile is set to Base."
-"test_000119","test_000002","3.5/codec_id, 4.1/Simple Profile",False,False,"An invalid stereo stream. The first mix uses a single stereo audio element encoded with LPCM. The second mix uses a single stereo audio element encoded with an imaginary `codec_id`. For convenience, the imaginary `codec_id` uses the same syntax as LPCM."
+"test_000119","test_000002","3.5/codec_id, 4.1/Simple Profile",False,True,"An invalid stereo stream. The first mix uses a single stereo audio element encoded with LPCM. The second mix uses a single stereo audio element encoded with an imaginary `codec_id`. For convenience, the imaginary `codec_id` uses the same syntax as LPCM."
 "test_000120","test_000119","3.6/audio_element_type, 4.1/Simple Profile",False,True,"An invalid IAMF stream. The first mix uses a single stereo audio element. The second mix presentation uses an `audio_element_type` which is reserved in Simple profile."
 "test_000121","test_000005","3.6/param_definition_type",True,True,"A Simple profile IAMF stream which uses a reserved parameter type that must be ignored."
 "test_000122","test_000119","3.6.2/loudspeaker_layout, 4.1/Simple Profile",False,True,"An invalid IAMF stream. The first mix uses a stereo audio element. The second mix uses a `loudspeaker_layout` which is reserved under Simple profile for the first layer."
@@ -172,5 +172,5 @@
 "test_000409","test_000400","7/IAMF Processing, 7.3/Mix Presentation, 7.3.1/Selecting a Mix Presentation",True,True,"A simple example of a stereo + stereo base profile IAMF stream has 2 Mix Presentation OBUs."
 "test_000500","test_000074","3.6.3/ambisonics_mono_config, 3.9/Audio Frame OBU Syntax and Semantics, 3.11.3/FLAC Specific",True,True,"A mixed first-order ambisonics IAMF stream encoded using FLAC and `ambisonics_mode` = `MONO`."
 "test_000501","test_000005","3.3/Reserved OBU Syntax and Semantics, 4/Profiles",True,True,"An IAMF stream which is backwards compatible with simple profile. It contains a reserved OBU inserted after the IA Sequence Header OBU which should be treated as a descriptor OBU."
-"test_000502","test_000409","3.7/num_sub_mixes",False,False,"An invalid IAMF stream with two Mix Presentation OBUs. The stream is invalid because one of the Mix Presentation OBUs has 0 sub-mixes."
+"test_000502","test_000409","3.7/num_sub_mixes",False,True,"An invalid IAMF stream with two Mix Presentation OBUs. The stream is invalid because one of the Mix Presentation OBUs has zero sub-mixes."
 "test_000503","test_000005","3.7.7/info_type, 3.7.7/info_type_size, 3.7.7/info_type_bytes",True,True,"An IAMF stream which uses a reserved bit in `info_type` which requires  the `info_type_size` and `info_type_bytes` fields to be used."