Merge pull request #664 from github/json-and-the-argowats

Interpret syntax imported from JSON

Author: robrix

.bazelignore

@@ -2,7 +2,6 @@ semantic/dist-newstyle
 semantic-analysis/dist-newstyle
 semantic-ast/dist-newstyle
 semantic-codeql/dist-newstyle
-semantic-core/dist-newstyle
 semantic-go/dist-newstyle
 semantic-java/dist-newstyle
 semantic-json/dist-newstyle

.github/workflows/haskell.yml

@@ -14,14 +14,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ghc: ["8.10.1"]
-        cabal: ["3.2.0.0"]
+        ghc: ["8.10"]
+        cabal: ["3.2"]
 
     steps:
     - uses: actions/checkout@v2
       if: github.event.action == 'opened' || github.event.action == 'synchronize' || github.event.ref == 'refs/heads/master'
 
-    - uses: actions/setup-haskell@v1
+    - uses: haskell/actions/setup@v1.2.2
       name: Setup Haskell
       with:
         ghc-version: ${{ matrix.ghc }}

build/common.bzl

@@ -119,7 +119,6 @@ def semantic_language_library(language, name, srcs, ts_package = "", nodetypes =
             "//:text",
             "//semantic-analysis",
             "//semantic-ast",
-            "//semantic-core",
             "//semantic-proto",
             "//semantic-scope-graph",
             "//semantic-source",

cabal.project

@@ -5,7 +5,6 @@ packages: semantic
           semantic-analysis
           semantic-ast
           semantic-codeql
-          semantic-core
           semantic-go
           semantic-java
           semantic-json

cabal.project.ci

@@ -5,7 +5,6 @@ packages: semantic
           semantic-analysis
           semantic-ast
           semantic-codeql
-          semantic-core
           semantic-go
           semantic-java
           semantic-json
@@ -34,9 +33,6 @@ package semantic-ast
 package semantic-codeql
   ghc-options: -Werror
 
-package semantic-core
-  ghc-options: -Werror
-
 package semantic-go
   ghc-options: -Werror
 

docs/adding-new-languages.md

@@ -8,14 +8,14 @@ Note that we recently transitioned the system to auto-generate strongly-typed AS
 
 1. **Find or write a [tree-sitter](https://tree-sitter.github.io) parser for your language.** The tree-sitter [organization page](https://github.com/tree-sitter) has a number of parsers beyond those we currently support in Semantic; look there first to make sure you're not duplicating work. The tree-sitter [documentation on creating parsers](http://tree-sitter.github.io/tree-sitter/creating-parsers) provides an exhaustive look at the process of developing and debugging tree-sitter parsers. Though we do not support grammars written with other toolkits such as [ANTLR](https://www.antlr.org), translating an ANTLR or other BNF-style grammar into a tree-sitter grammar is usually straightforward.
 2. **Create a Haskell library providing an interface to that C source.** The [`haskell-tree-sitter`](https://github.com/tree-sitter/haskell-tree-sitter) repository provides a Cabal package for each supported language. You can find an example of a pull request to add such a package [here](https://github.com/tree-sitter/haskell-tree-sitter/pull/276/files), and a file providing:
-    - A bridged (via the FFI) reference to the toplevel parser in the generated file must be provided ([example](https://github.com/tree-sitter/haskell-tree-sitter/blob/master/tree-sitter-json/TreeSitter/JSON.hs#L11)). 
-    - A way to retrieve [`tree-sitter` data](https://github.com/tree-sitter/haskell-tree-sitter/blob/master/tree-sitter-json/TreeSitter/JSON.hs#L13-L14) used to auto-generate syntax datatypes using the following steps. During parser generation, tree-sitter produces a `node-types.json` file that captures the structure of a language's grammar. The autogeneration described below in Step 4 derives datatypes based on this structural representation. The `node-types.json` is a data file in `haskell-tree-sitter` that gets installed with the package. The function `getNodeTypesPath :: IO FilePath` is defined to access in the contents of this file, using `getDataFileName :: FilePath -> IO FilePath`, which is defined in the autogenerated `Paths_` module. 
+    - A bridged (via the FFI) reference to the toplevel parser in the generated file must be provided ([example](https://github.com/tree-sitter/haskell-tree-sitter/blob/master/tree-sitter-json/TreeSitter/JSON.hs#L11)).
+    - A way to retrieve [`tree-sitter` data](https://github.com/tree-sitter/haskell-tree-sitter/blob/master/tree-sitter-json/TreeSitter/JSON.hs#L13-L14) used to auto-generate syntax datatypes using the following steps. During parser generation, tree-sitter produces a `node-types.json` file that captures the structure of a language's grammar. The autogeneration described below in Step 4 derives datatypes based on this structural representation. The `node-types.json` is a data file in `haskell-tree-sitter` that gets installed with the package. The function `getNodeTypesPath :: IO FilePath` is defined to access in the contents of this file, using `getDataFileName :: FilePath -> IO FilePath`, which is defined in the autogenerated `Paths_` module.
 3. **Create a Haskell library in Semantic to auto-generate precise ASTs.** Create a `semantic-[LANGUAGE]` package. This is an example of [`semantic-python`](https://github.com/github/semantic/tree/master/semantic-python)). Each package needs to provide the following API surfaces:
     - `Language.[LANGUAGE].AST` - Derives Haskell datatypes from a language and its `node-types.json` file ([example](https://github.com/github/semantic/blob/master/semantic-python/src/Language/Python/AST.hs)).
     - `Language.[LANGUAGE].Grammar` - Provides statically-known rules corresponding to symbols in the grammar for each syntax node, generated with the `mkStaticallyKnownRuleGrammarData` Template Haskell splice ([example](https://github.com/github/semantic/blob/master/semantic-python/src/Language/Python/Grammar.hs)).
     - `Language.[LANGUAGE]` - Semantic functionality for programs in a language ([example](https://github.com/github/semantic/blob/master/semantic-python/src/Language/Python.hs)).
     - `Language.[LANGUAGE].Tags` - Computes tags for code nav definitions and references found in source ([example](https://github.com/github/semantic/blob/master/semantic-python/src/Language/Python/Tags.hs)).
-5. **Add tests for precise ASTs, tagging and graphing, and evaluating code written in that language.** Because tree-sitter grammars often change, we require extensive testing so as to avoid the unhappy situation of bitrotted languages that break as soon as a new grammar comes down the line. Here are examples of tests for [precise ASTs](https://github.com/github/semantic/blob/master/semantic-python/test/PreciseTest.hs), [tagging](https://github.com/github/semantic/blob/master/test/Tags/Spec.hs), and [graphing](https://github.com/github/semantic/blob/master/semantic-python/test-graphing/GraphTest.hs). 
+5. **Add tests for precise ASTs, tagging and graphing, and evaluating code written in that language.** Because tree-sitter grammars often change, we require extensive testing so as to avoid the unhappy situation of bitrotted languages that break as soon as a new grammar comes down the line. Here are examples of tests for [precise ASTs](https://github.com/github/semantic/blob/master/semantic-python/test/PreciseTest.hs), [tagging](https://github.com/github/semantic/blob/master/test/Tags/Spec.hs), and [graphing](https://github.com/github/semantic/blob/master/semantic-python/test-graphing/GraphTest.hs).
 
 To summarize, each interaction made possible by the Semantic CLI corresponds to one (or more) of the above steps:
 
@@ -30,4 +30,4 @@ To summarize, each interaction made possible by the Semantic CLI corresponds to
 
 **This sounds hard.** You're right! It is currently a lot of work: just because the Semantic architecture is extensible in the expression-problem manner does not mean that adding new support is trivial.
 
-**What recent changes have been made?** The Semantic authors have introduced a new architecture for language support and parsing, one that dispenses with the [assignment](https://github.com/github/semantic/blob/master/docs/assignment.md) step altogether. The `semantic-ast` package generates Haskell data types from tree-sitter grammars; these types are then translated into the [Semantic core language](https://github.com/github/semantic/blob/master/semantic-core/src/Data/Core.hs); all evaluators will then be written in terms of the Core language. As compared with the [historic process]() used to add new languages, these changes entire obviate the process of 1) assigning types into an open-union of syntax functors, and 2) implementing `Evaluatable` instances and adding value effects to describe the control flow of your language. 
+**What recent changes have been made?** The Semantic authors have introduced a new architecture for language support and parsing, one that dispenses with the [assignment](https://github.com/github/semantic/blob/master/docs/assignment.md) step altogether. The `semantic-ast` package generates Haskell data types from tree-sitter grammars. As compared with the [historic process]() used to add new languages, these changes entire obviate the process of 1) assigning types into an open-union of syntax functors, and 2) implementing `Evaluatable` instances and adding value effects to describe the control flow of your language.

script/ghci-flags

@@ -58,7 +58,6 @@ function flags {
   echo "-isemantic-ast/src"
   echo "-isemantic-codeql/src"
   echo "-isemantic-codeql/test"
-  echo "-isemantic-core/src"
   echo "-isemantic-go/src"
   echo "-isemantic-java/src"
   echo "-isemantic-json/src"

script/ghci-flags-dependencies

@@ -10,7 +10,6 @@ echo "cabal.project"
 echo "semantic.cabal"
 echo "semantic-analysis/semantic-analysis.cabal"
 echo "semantic-ast/semantic-ast.cabal"
-echo "semantic-core/semantic-core.cabal"
 echo "semantic-tags/semantic-tags.cabal"
 echo "semantic-go/semantic-go.cabal"
 echo "semantic-java/semantic-java.cabal"

semantic-analysis/BUILD.bazel

@@ -18,9 +18,10 @@ load(
 haskell_library(
     name = "semantic-analysis",
     srcs = glob(["src/**/*.hs"]),
-    compiler_flags = GHC_FLAGS + ["-XOverloadedStrings"],
+    compiler_flags = GHC_FLAGS,
     deps = [
         "//:base",
+        "//:bytestring",
         "//:containers",
         "//:filepath",
         "//:text",
@@ -30,5 +31,6 @@ haskell_library(
         "@stackage//:fused-effects",
         "@stackage//:hashable",
         "@stackage//:pathtype",
+        "@stackage//:vector",
     ],
 )

semantic-analysis/python.tsg

@@ -0,0 +1,82 @@
+; tree-sitter-graph definitions mapping Python ASTs to an analyzable IR.
+
+(module) @this
+{
+  node @this.node
+  attr (@this.node) type = "module"
+}
+
+(identifier) @id
+{
+  node @id.node
+}
+
+(string) @this
+{
+  node @this.node
+  attr (@this.node) type = "string"
+  attr (@this.node) text = (source-text @this)
+}
+
+(true) @this
+{
+  node @this.node
+  attr (@this.node) type = "true"
+}
+
+(false) @this
+{
+  node @this.node
+  attr (@this.node) type = "false"
+}
+
+(print_statement argument: (_) @arg) @this
+{
+  node @this.node
+  attr (@this.node) type = "print"
+  edge @this.node -> @arg.node
+}
+
+(raise_statement (_) @arg) @this
+{
+  node @this.node
+  attr (@this.node) type = "throw"
+  edge @this.node -> @arg.node
+}
+
+(block (_)* @children) @this
+{
+  node @this.node
+  attr (@this.node) type = "block"
+  for child in @children {
+    edge @this.node -> child.node
+  }
+}
+
+(else_clause body: (_) @body) @this
+{
+  let @this.node = @body.node
+}
+
+(if_statement (_)) @this {
+  node @this.node
+  attr (@this.node) type = "if"
+}
+
+(if_statement condition: (_) @cond consequence: (_) @then) @this {
+  edge @this.node -> @cond.node
+  attr (@this.node -> @cond.node) type = "condition"
+  edge @this.node -> @then.node
+  attr (@this.node -> @then.node) type = "consequence"
+}
+
+(if_statement alternative: (_) @else) @this
+{
+  edge @this.node -> @else.node
+  attr (@this.node -> @else.node) type = "alternative"
+}
+
+(module (_) @child) @this
+{
+  edge @this.node -> @child.node
+}