Merge branch 'var-date-time'

Fixes #11

Author: waveform80

docs/images/types.png

@@ -20,7 +20,7 @@ Synopsis
             [--hide-pattern] [--show-pattern]
             [--hide-range] [--show-range {hidden,limits,median,quartiles,graph}]
             [--hide-samples] [--show-samples]
-            [--min-timestamp WHEN] [--max-timestamp WHEN]
+            [--min-timestamp WHEN] [--max-timestamp WHEN] [--epoch WHEN ]
             [--max-numeric-len LEN] [--sample-bytes SIZE]
             [--strip-whitespace] [--no-strip-whitespace]
             [--csv-format FIELD[QUOTE]] [--yaml-safe] [--no-yaml-safe]
@@ -145,6 +145,14 @@ Optional Arguments
     absolute timestamp (in ISO-8601 format) or a duration to be added to the
     current timestamp
 
+.. option:: --epoch WHEN
+
+    The epoch from which datetimes are measured. Can be specified as an
+    absolute timestamp (in ISO-8601 format: YYYY-mm-ddTHH:MM:SS), or one of the
+    special strings, "unix" (which is equivalent to 1970-01-01) or
+    "excel" (which is roughly equivalent to 1900-01-01, with some adjustments).
+    The default is "unix"
+
 .. option:: --max-numeric-len LEN
 
     The maximum number of characters that a number, integer or floating-point,

docs/manual.rst

@@ -433,8 +433,9 @@ Date Handling
 The other important switches are those used in the detection of dates encoded
 as numbers: :option:`structa --min-timestamp` and :option:`structa
 --max-timestamp`. When dates are encoded as (potentially fractional)
-day-offsets from the UNIX epoch (the 1st January, 1970), how does structa
-determine that it's looking at a set of dates rather than a set of numbers?
+day-offsets from some :option:`structa --epoch` (which defaults to the UNIX
+epoch, i.e. the 1st January, 1970), how does structa determine that it's
+looking at a set of dates rather than a set of numbers?
 
 In a typical set of (arbitrary) numbers, it's quite normal to find "0" or "1"
 commonly represented, or for the set of numbers to span over a large range

docs/tutorial_basic.rst

@@ -25,6 +25,7 @@ classifiers =
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
+    Programming Language :: Python :: Implementation :: PyPy
 
 [options]
 packages = find:

setup.cfg

@@ -158,6 +158,16 @@ class Analyzer:
         The maximum timestamp to use when determining whether floating point
         values potentially represent epoch-based datetime values.
 
+    :type epoch: datetime.datetime or None
+    :param epoch:
+        The epoch to use when converting numbers to datetime values. Defaults
+        to the UNIX epoch (1st January, 1970).
+
+    :type epoch_unit: datetime.timedelta or None
+    :param epoch_unit:
+        The unit of time used in numeric representations of datetime values.
+        Defaults to 1 second.
+
     :type progress: object or None
     :param progress:
         If specificed, must be an object with ``update`` and ``reset`` methods
@@ -168,22 +178,32 @@ def __init__(self, *, bad_threshold=Fraction(2, 100),
                  empty_threshold=Fraction(98, 100),
                  null_threshold=Fraction(98, 100), field_threshold=20,
                  merge_threshold=Fraction(50, 100), max_numeric_len=30,
-                 strip_whitespace=False, min_timestamp=None,
-                 max_timestamp=None, progress=None):
+                 strip_whitespace=False,
+                 min_timestamp=None, max_timestamp=None,
+                 epoch=datetime.utcfromtimestamp(0),
+                 epoch_unit=timedelta(seconds=1),
+                 progress=None):
         self.bad_threshold = bad_threshold
         self.empty_threshold = empty_threshold
         self.null_threshold = null_threshold
         self.field_threshold = field_threshold
         self.merge_threshold = merge_threshold
         self.max_numeric_len = max_numeric_len
         self.strip_whitespace = strip_whitespace
+        unix_epoch = datetime.utcfromtimestamp(0)
+        self.timestamp_offset = (epoch - unix_epoch).total_seconds()
+        self.timestamp_scale = epoch_unit.total_seconds()
         now = datetime.now()
         if min_timestamp is None:
             min_timestamp = now - relativedelta(years=20)
         if max_timestamp is None:
             max_timestamp = now + relativedelta(years=10)
-        self.min_timestamp = min_timestamp.timestamp()
-        self.max_timestamp = max_timestamp.timestamp()
+        self.min_timestamp = (
+            min_timestamp.timestamp() - self.timestamp_offset
+        ) / self.timestamp_scale
+        self.max_timestamp = (
+            max_timestamp.timestamp() - self.timestamp_offset
+        ) / self.timestamp_scale
         self._progress = progress
 
     @property
@@ -732,7 +752,8 @@ def _match_possible_datetime(self, pattern):
                 isinstance(pattern, (Int, Float)) and
                 in_range(pattern.values.min) and
                 in_range(pattern.values.max)):
-            return DateTime.from_numbers(pattern)
+            return DateTime.from_numbers(pattern, offset=self.timestamp_offset,
+                                         scale=self.timestamp_scale)
         elif (
                 isinstance(pattern, StrRepr) and (
                     (
@@ -743,6 +764,7 @@ def _match_possible_datetime(self, pattern):
                 ) and
                 in_range(pattern.content.values.min) and
                 in_range(pattern.content.values.max)):
-            return DateTime.from_numbers(pattern)
+            return DateTime.from_numbers(pattern, offset=self.timestamp_offset,
+                                         scale=self.timestamp_scale)
         else:
             return pattern

structa/analyzer.py

@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 import re
+from datetime import timedelta
 
 from dateutil.parser import parse
 from dateutil.relativedelta import relativedelta
@@ -84,18 +85,22 @@ def parse_bool(s, false='0', true='1'):
 }
 
 
-def parse_duration(s):
+def parse_duration(s, delta_type=relativedelta):
     """
-    Convert the string *s* to a :class:`~dateutil.relativedelta.relativedelta`.
-    The string must consist of white-space and/or comma separated values which
-    are a number followed by a suffix indicating duration. For example:
+    Convert the string *s* to a :class:`~dateutil.relativedelta.relativedelta`
+    (by default) or a :class:`~datetime.timedelta` if requested by
+    *delta_type*. The string must consist of white-space and/or comma separated
+    values which are a number followed by a suffix indicating duration. For
+    example:
 
         >>> parse_duration('1s')
         relativedelta(seconds=+1)
         >>> parse_duration('5 minutes, 30 seconds')
         relativedelta(minutes=+5, seconds=+30)
         >>> parse_duration('1 year')
         relativedelta(years=+1)
+        >>> parse_duration('1 week, 1 day', delta_type=datetime.timedelta)
+        timedelta(days=8)
 
     Note that some suffixes like "m" can be ambiguous; using common
     abbreviations should avoid ambiguity:
@@ -112,6 +117,9 @@ def parse_duration(s):
     * *Microseconds*: microseconds, microsecond, microsec, micros, micro,
       useconds, usecond, usecs, usec, us, µseconds, µsecond, µsecs, µsec, µs
 
+    * *Milliseconds*: milliseconds, millisecond, millisec, millis, milli,
+      mseconds, msecond, msecs, msec, ms
+
     * *Seconds*: seconds, second, secs, sec, s
 
     * *Minutes*: minutes, minute, mins, min, mi
@@ -122,35 +130,50 @@ def parse_duration(s):
 
     * *Weeks*: weeks, week, wks, wk, w
 
-    * *Months*: months, month, mons, mon, mths, mth, m
+    * *Months*: months, month, mons, mon, mths, mth, m (relativedelta only)
 
-    * *Years*: years, year, yrs, yr, y
+    * *Years*: years, year, yrs, yr, y (relativedelta only)
 
     If conversion fails, :exc:`ValueError` is raised.
     """
-    spans = {span: 0 for span in _SPANS}
+    assert delta_type in (relativedelta, timedelta)
+    spans = {}
     t = s
     for span, regex in _SPANS.items():
+        if delta_type is timedelta and span in ('months', 'years'):
+            continue
         m = regex.search(t)
         if m:
-            spans[span] += int(m.group('num'))
+            spans[span] = spans.get(span, 0) + int(m.group('num'))
             t = (t[:m.start(0)] + t[m.end(0):]).strip(' \t\n,')
             if not t:
                 break
     if t:
         raise ValueError('invalid duration {}'.format(s))
-    spans['microseconds'] += spans.pop('milliseconds') * 1000
-    return relativedelta(**spans)
+    if delta_type is relativedelta:
+        spans['microseconds'] = (
+            spans.get('microseconds', 0) +
+            (spans.pop('milliseconds', 0) * 1000))
+    return delta_type(**spans)
+
+
+def parse_timestamp(s):
+    """
+    Convert the string *s* to a :class:`~datetime.datetime`. A
+    :exc:`ValueError` is raised if *s* is not a valid datetime representation.
+    """
+    return parse(s)
 
 
-def parse_duration_or_timestamp(s):
+def parse_duration_or_timestamp(s, delta_type=relativedelta):
     """
     Convert the string *s* to a :class:`~datetime.datetime` or a
-    :class:`~dateutil.relativedelta.relativedelta`. Duration conversion is
-    attempted to and, if this fails, date-time conversion is attempted. A
-    :exc:`ValueError` is raised if both conversions fail.
+    :class:`~dateutil.relativedelta.relativedelta` (or
+    :class:`~datetime.timedelta` if *duration_type* so specifies). Duration
+    conversion is attempted to and, if this fails, date-time conversion is
+    attempted. A :exc:`ValueError` is raised if both conversions fail.
     """
     try:
-        return parse_duration(s)
+        return parse_duration(s, delta_type=delta_type)
     except ValueError:
-        return parse(s)
+        return parse_timestamp(s)

structa/conversions.py

@@ -6,7 +6,7 @@
 
 from math import log
 from itertools import tee
-from datetime import datetime
+from datetime import datetime, timedelta
 
 
 def pairwise(iterable):
@@ -153,3 +153,29 @@ def format_sample(value):
         }[type(value)]()
     except KeyError:
         raise ValueError('invalid type for value {!r}'.format(value))
+
+
+def format_timestamp_numrepr(offset, scale):
+    delta = timedelta(seconds=scale)
+    simple = {
+        timedelta(**{name: 1}): name
+        for name in (
+            'microseconds',
+            'milliseconds',
+            'seconds',
+            'minutes',
+            'hours',
+            'days',
+            'weeks',
+        )
+    }
+    if offset % 86400:
+        epoch = datetime.utcfromtimestamp(offset).isoformat()
+    else:
+        epoch = datetime.utcfromtimestamp(offset).date().isoformat()
+    try:
+        return '{unit} since {epoch}'.format(unit=simple[delta], epoch=epoch)
+    except KeyError:
+        return 'seconds since {epoch} {op} {scale:g}'.format(
+            epoch=epoch, op=('*', '/')[scale >= 1],
+            scale=scale if scale >= 1 else (1 / scale))

structa/format.py

@@ -15,8 +15,13 @@
 
 from .collections import Counter, FrozenCounter
 from .conversions import try_conversion, parse_bool
-from .format import format_int, format_repr, format_sample
 from .xml import ElementFactory, xml, merge_siblings
+from .format import (
+    format_int,
+    format_repr,
+    format_sample,
+    format_timestamp_numrepr,
+)
 
 
 tag = ElementFactory()
@@ -905,11 +910,15 @@ def from_strings(cls, iterable, pattern, bad_threshold=0):
             pattern=pattern)
 
     @classmethod
-    def from_numbers(cls, pattern):
+    def from_numbers(cls, pattern, offset=0, scale=1):
         """
         Class method for constructing an instance wrapped in a :class:`NumRepr`
         to indicate a numeric representation of a set of timestamps (e.g. day
-        offset from the UNIX epoch).
+        offset from the UNIX epoch). A different epoch may be specified as a
+        numeric *offset*, and a different epoch *scale* as a numeric number of
+        seconds). The default offset and scale are 0 and 1 which is equivalent
+        to a seconds offset from the UNIX epoch (i.e. a traditional UNIX
+        timestamp).
 
         Constructed with an *sample* of number, a *pattern* (which can be a
         :class:`StrRepr` instance if the numbers are themselves represented as
@@ -923,8 +932,10 @@ def from_numbers(cls, pattern):
             num_pattern = pattern
         dt_counter = Counter()
         for value, count in num_pattern.values.sample.items():
-            dt_counter[datetime.fromtimestamp(value)] = count
-        result = NumRepr(cls(dt_counter), pattern=num_pattern.__class__)
+            dt_value = datetime.utcfromtimestamp((value * scale) + offset)
+            dt_counter[dt_value] = count
+        result = NumRepr(cls(dt_counter), pattern=(
+            num_pattern.__class__, scale, offset))
         if isinstance(pattern, StrRepr):
             return pattern.with_content(result)
         else:
@@ -1174,12 +1185,12 @@ def validate(self, value):
             value = parse_bool(value, false, true)
         elif isinstance(self.content, Int) or (
             isinstance(self.content, NumRepr) and
-            self.content.pattern is Int
+            self.content.pattern[0] is Int
         ):
             value = int(value, base=self.int_bases[self.pattern])
         elif isinstance(self.content, Float) or (
             isinstance(self.content, NumRepr) and
-            self.content.pattern is Float
+            self.content.pattern[0] is Float
         ):
             assert self.pattern == 'f'
             value = float(value)
@@ -1199,36 +1210,52 @@ class NumRepr(Repr):
     __slots__ = ()
 
     def __str__(self):
-        if self.pattern is Int:
-            template = 'int of {self.content}'
-        elif self.pattern is Float:
-            template = 'float of {self.content}'
-        else:
+        type_, scale, offset = self.pattern
+        try:
+            type_name = {Int: 'int', Float: 'float'}[type_]
+        except KeyError:
             assert False, 'str(num-repr) of {self.content!r}'.format(self=self)
-        return template.format(self=self)
+        return '{type_name} {numrepr} of {self.content}'.format(
+            self=self, type_name=type_name,
+            numrepr=format_timestamp_numrepr(offset, scale))
 
     def __xml__(self):
-        if self.pattern is Int:
-            return tag.intof(xml(self.content))
-        elif self.pattern is Float:
-            return tag.floatof(xml(self.content))
+        type_, scale, offset = self.pattern
+        if type_ is Int:
+            return tag.intof(xml(self.content), scale=scale, offset=offset)
+        elif type_ is Float:
+            return tag.floatof(xml(self.content), scale=scale, offset=offset)
         else:
             assert False, 'xml(num-repr) of {self.content!r}'.format(self=self)
 
     def __add__(self, other):
         if self == other:
-            if self.pattern is Float or other.pattern is Float:
-                pattern = Float
+            self_type, self_scale, self_offset = self.pattern
+            other_type, other_scale, other_offset = other.pattern
+            if self_type is Float or other_type is Float:
+                add_type = Float
             else:
-                pattern = Int
-            return NumRepr(self.content + other.content, pattern)
+                add_type = Int
+            return NumRepr(
+                self.content + other.content,
+                (add_type, self_scale, self_offset))
         return NotImplemented
 
+    def __eq__(self, other):
+        if not isinstance(other, NumRepr):
+            return NotImplemented
+        if super().__eq__(other) is not True:
+            return False
+        self_type, self_scale, self_offset = self.pattern
+        other_type, other_scale, other_offset = other.pattern
+        return (self_scale == other_scale) and (self_offset == other_offset)
+
     def validate(self, value):
         if not isinstance(value, Real):
             raise TypeError('{value!r} is not a number'.format(value=value))
         if isinstance(self.content, DateTime):
-            value = datetime.fromtimestamp(value)
+            type_, scale, offset = self.pattern
+            value = datetime.utcfromtimestamp((value * scale) + offset)
         else:
             assert False, (
                 'validating num-repr of {self.content!r}'.format(self=self))