Migrate to go-enry new version (#10906)

mj
Lauris BH 4 years ago committed by GitHub
parent 7a67bcc204
commit 4dc62dadce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -43,6 +43,7 @@ require (
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
github.com/gliderlabs/ssh v0.2.2
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a // indirect
github.com/go-enry/go-enry/v2 v2.3.0
github.com/go-git/go-billy/v5 v5.0.0
github.com/go-git/go-git/v5 v5.0.0
github.com/go-openapi/jsonreference v0.19.3 // indirect
@ -93,7 +94,6 @@ require (
github.com/sergi/go-diff v1.1.0
github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b // indirect
github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
github.com/src-d/enry/v2 v2.1.0
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 // indirect
github.com/stretchr/testify v1.4.0
github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect

@ -183,6 +183,10 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a h1:FQqo
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/go-enry/go-enry/v2 v2.3.0 h1:o8KwgY6uSplysrIpj+Y42J/xGPp90ogVpxE2Z3s8Unk=
github.com/go-enry/go-enry/v2 v2.3.0/go.mod h1:+xFJwbqWi15bvqFHb2ELUWVRKFQtwB61+sDrkvvxxGI=
github.com/go-enry/go-oniguruma v1.2.0 h1:oBO9XC1IDT9+AoWW5oFsa/7gFeOPacEqDbyXZKWXuDs=
github.com/go-enry/go-oniguruma v1.2.0/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4=
github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM=
@ -560,10 +564,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/src-d/enry/v2 v2.1.0 h1:z1L8t+B8bh3mmjPkJrgOTnVRpFGmTPJsplHX9wAn6BI=
github.com/src-d/enry/v2 v2.1.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj0t028=
github.com/src-d/go-oniguruma v1.1.0 h1:EG+Nm5n2JqWUaCjtM0NtutPxU7ZN5Tp50GWrrV8bTww=
github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM=
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s=
github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -829,6 +829,8 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

@ -10,7 +10,7 @@ import (
"code.gitea.io/gitea/modules/timeutil"
"github.com/src-d/enry/v2"
"github.com/go-enry/go-enry/v2"
)
// LanguageStat describes language statistics of a repository

@ -7,7 +7,7 @@ package analyze
import (
"path/filepath"
"github.com/src-d/enry/v2"
"github.com/go-enry/go-enry/v2"
)
// GetCodeLanguageWithCallback detects code language based on file name and content using callback

@ -12,10 +12,10 @@ import (
"code.gitea.io/gitea/modules/analyze"
"github.com/go-enry/go-enry/v2"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/src-d/enry/v2"
)
const fileSizeLimit int64 = 16 * 1024 * 1024

@ -30,7 +30,7 @@ import (
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search/query"
"github.com/ethantkoenig/rupture"
"github.com/src-d/enry/v2"
"github.com/go-enry/go-enry/v2"
)
const unicodeNormalizeName = "unicodeNormalize"

@ -1,4 +1,4 @@
.linguist
.linguist*
benchmarks/output
.ci
Makefile.main
@ -9,3 +9,4 @@ build/
vendor/
java/lib/
.vscode/
.venv

@ -1,8 +1,9 @@
dist: trusty
language: go
go:
- '1.12.x'
- '1.11.x'
- '1.14.x'
- '1.13.x'
env:
global:
- GO_VERSION_FOR_JVM='1.11.x'
@ -12,8 +13,6 @@ env:
matrix:
- ONIGURUMA=0
- ONIGURUMA=1
matrix:
fast_finish: true
stages:
- name: test
@ -22,7 +21,6 @@ stages:
- name: publish
if: tag IS present
stage: test
install:
- >
if [[ "${ONIGURUMA}" -gt 0 ]]; then
@ -36,7 +34,7 @@ install:
sudo dpkg -i "libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
fi;
script:
- make test-coverage
- go test ./...
after_success:
- bash <(curl -s https://codecov.io/bash)
@ -98,7 +96,7 @@ jobs:
- sudo apt-get install -y --no-install-recommends clang g++ gcc gcc-multilib libc6-dev libc6-dev-i386 mingw-w64 patch xz-utils
- cd ${HOME}
- curl -sfSL ${OSXCROSS_URL} | tar -C ${HOME} -xzf -
- cd $GOPATH/src/github.com/src-d/enry
- cd $GOPATH/src/github.com/bzz/enry
script: make darwin-shared
deploy:
provider: releases

@ -1,3 +1,4 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
@ -178,7 +179,7 @@
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
@ -186,7 +187,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2017 Sourced Technologies, S.L.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -198,4 +199,4 @@
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
limitations under the License.

@ -1,26 +1,3 @@
# Package configuration
PROJECT = enry
COMMANDS = cmd/enry
# Including ci Makefile
CI_REPOSITORY ?= https://github.com/src-d/ci.git
CI_BRANCH ?= v1
CI_PATH ?= .ci
MAKEFILE := $(CI_PATH)/Makefile.main
$(MAKEFILE):
git clone --quiet --depth 1 -b $(CI_BRANCH) $(CI_REPOSITORY) $(CI_PATH);
-include $(MAKEFILE)
# Docsrv: configure the languages whose api-doc can be auto generated
LANGUAGES = go
# Docs: do not edit this
DOCS_REPOSITORY := https://github.com/src-d/docs
SHARED_PATH ?= $(shell pwd)/.docsrv-resources
DOCS_PATH ?= $(SHARED_PATH)/.docs
$(DOCS_PATH)/Makefile.inc:
git clone --quiet --depth 1 $(DOCS_REPOSITORY) $(DOCS_PATH);
-include $(DOCS_PATH)/Makefile.inc
LINGUIST_PATH = .linguist
# shared objects
@ -29,6 +6,7 @@ LINUX_DIR=$(RESOURCES_DIR)/linux-x86-64
LINUX_SHARED_LIB=$(LINUX_DIR)/libenry.so
DARWIN_DIR=$(RESOURCES_DIR)/darwin
DARWIN_SHARED_LIB=$(DARWIN_DIR)/libenry.dylib
STATIC_LIB=$(RESOURCES_DIR)/libenry.a
HEADER_FILE=libenry.h
NATIVE_LIB=./shared/enry.go
@ -79,4 +57,10 @@ $(LINUX_SHARED_LIB):
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=c-shared -o $(LINUX_SHARED_LIB) $(NATIVE_LIB) && \
mv $(LINUX_DIR)/$(HEADER_FILE) $(RESOURCES_DIR)/$(HEADER_FILE)
static: $(STATIC_LIB)
$(STATIC_LIB):
CGO_ENABLED=1 go build -buildmode=c-archive -o $(STATIC_LIB) $(NATIVE_LIB)
.PHONY: benchmarks benchmarks-samples benchmarks-slow

@ -0,0 +1,303 @@
# go-enry [![GoDoc](https://godoc.org/github.com/go-enry/go-enry?status.svg)](https://pkg.go.dev/github.com/go-enry/go-enry/v2) [![Test](https://github.com/go-enry/go-enry/workflows/Test/badge.svg)](https://github.com/go-enry/go-enry/actions?query=workflow%3ATest+branch%3Amaster) [![codecov](https://codecov.io/gh/go-enry/go-enry/branch/master/graph/badge.svg)](https://codecov.io/gh/go-enry/go-enry)
Programming language detector and toolbox to ignore binary or vendored files. *enry*, started as a port to _Go_ of the original [Linguist](https://github.com/github/linguist) _Ruby_ library, that has an improved *2x performance*.
* [CLI](#cli)
* [Library](#library)
* [Use cases](#use-cases)
* [By filename](#by-filename)
* [By text](#by-text)
* [By file](#by-file)
* [Filtering](#filtering-vendoring-binaries-etc)
* [Coloring](#language-colors-and-groups)
* [Languages](#languages)
* [Go](#go)
* [Java bindings](#java-bindings)
* [Python bindings](#python-bindings)
* [Divergences from linguist](#divergences-from-linguist)
* [Benchmarks](#benchmarks)
* [Why Enry?](#why-enry)
* [Development](#development)
* [Sync with github/linguist upstream](#sync-with-githublinguist-upstream)
* [Misc](#misc)
* [License](#license)
# CLI
The CLI binary is hosted in a separate repository [go-enry/enry](https://github.com/go-enry/enry).
# Library
*enry* is also a Go library for guessing a programming language that exposes API through FFI to multiple programming environments.
## Use cases
*enry* guesses a programming language using a sequence of matching *strategies* that are
applied progressively to narrow down the possible options. Each *strategy* varies on the type
of input data that it needs to make a decision: file name, extension, the first line of the file, the full content of the file, etc.
Depending on available input data, enry API can be roughly divided into the next categories or use cases.
### By filename
Next functions require only a name of the file to make a guess:
- `GetLanguageByExtension` uses only file extension (wich may be ambiguous)
- `GetLanguageByFilename` useful for cases like `.gitignore`, `.bashrc`, etc
- all [filtering helpers](#filtering)
Please note that such guesses are expected not to be very accurate.
### By text
To make a guess only based on the content of the file or a text snippet, use
- `GetLanguageByShebang` reads only the first line of text to identify the [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)).
- `GetLanguageByModeline` for cases when Vim/Emacs modeline e.g. `/* vim: set ft=cpp: */` may be present at a head or a tail of the text.
- `GetLanguageByClassifier` uses a Bayesian classifier trained on all the `./samples/` from Linguist.
It usually is a last-resort strategy that is used to disambiguate the guess of the previous strategies, and thus it requires a list of "candidate" guesses. One can provide a list of all known languages - keys from the `data.LanguagesLogProbabilities` as possible candidates if more intelligent hypotheses are not available, at the price of possibly suboptimal accuracy.
### By file
The most accurate guess would be one when both, the file name and the content are available:
- `GetLanguagesByContent` only uses file extension and a set of regexp-based content heuristics.
- `GetLanguages` uses the full set of matching strategies and is expected to be most accurate.
### Filtering: vendoring, binaries, etc
*enry* expose a set of file-level helpers `Is*` to simplify filtering out the files that are less interesting for the purpose of source code analysis:
- `IsBinary`
- `IsVendor`
- `IsConfiguration`
- `IsDocumentation`
- `IsDotFile`
- `IsImage`
### Language colors and groups
*enry* exposes function to get language color to use for example in presenting statistics in graphs:
- `GetColor`
- `GetLanguageGroup` can be used to group similar languages together e.g. for `Less` this function will return `CSS`
## Languages
### Go
In a [Go module](https://github.com/golang/go/wiki/Modules),
import `enry` to the module by running:
```sh
go get github.com/go-enry/go-enry/v2
```
The rest of the examples will assume you have either done this or fetched the
library into your `GOPATH`.
```go
// The examples here and below assume you have imported the library.
import "github.com/go-enry/go-enry/v2"
lang, safe := enry.GetLanguageByExtension("foo.go")
fmt.Println(lang, safe)
// result: Go true
lang, safe := enry.GetLanguageByContent("foo.m", []byte("<matlab-code>"))
fmt.Println(lang, safe)
// result: Matlab true
lang, safe := enry.GetLanguageByContent("bar.m", []byte("<objective-c-code>"))
fmt.Println(lang, safe)
// result: Objective-C true
// all strategies together
lang := enry.GetLanguage("foo.cpp", []byte("<cpp-code>"))
// result: C++ true
```
Note that the returned boolean value `safe` is `true` if there is only one possible language detected.
A plural version of the same API allows getting a list of all possible languages for a given file.
```go
langs := enry.GetLanguages("foo.h", []byte("<cpp-code>"))
// result: []string{"C", "C++", "Objective-C}
langs := enry.GetLanguagesByExtension("foo.asc", []byte("<content>"), nil)
// result: []string{"AGS Script", "AsciiDoc", "Public Key"}
langs := enry.GetLanguagesByFilename("Gemfile", []byte("<content>"), []string{})
// result: []string{"Ruby"}
```
### Java bindings
Generated Java bindings using a C shared library and JNI are available under [`java`](https://github.com/go-enry/go-enry/blob/master/java).
A library is published on Maven as [tech.sourced:enry-java](https://mvnrepository.com/artifact/tech.sourced/enry-java) for macOS and linux platforms. Windows support is planned under [src-d/enry#150](https://github.com/src-d/enry/issues/150).
### Python bindings
Generated Python bindings using a C shared library and cffi are WIP under [src-d/enry#154](https://github.com/src-d/enry/issues/154).
A library is going to be published on pypi as [enry](https://pypi.org/project/enry/) for
macOS and linux platforms. Windows support is planned under [src-d/enry#150](https://github.com/src-d/enry/issues/150).
Divergences from Linguist
------------
The `enry` library is based on the data from `github/linguist` version **v7.9.0**.
Parsing [linguist/samples](https://github.com/github/linguist/tree/master/samples) the following `enry` results are different from the Linguist:
* [Heuristics for ".es" extension](https://github.com/github/linguist/blob/e761f9b013e5b61161481fcb898b59721ee40e3d/lib/linguist/heuristics.yml#L103) in JavaScript could not be parsed, due to unsupported backreference in RE2 regexp engine.
* [Heuristics for ".rno" extension](https://github.com/github/linguist/blob/3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d/lib/linguist/heuristics.yml#L365) in RUNOFF could not be parsed, due to unsupported lookahead in RE2 regexp engine.
* [Heuristics for ".inc" extension](https://github.com/github/linguist/blob/f0e2d0d7f1ce600b2a5acccaef6b149c87d8b99c/lib/linguist/heuristics.yml#L222) in NASL could not be parsed, due to unsupported possessive quantifier in RE2 regexp engine.
* As of [Linguist v5.3.2](https://github.com/github/linguist/releases/tag/v5.3.2) it is using [flex-based scanner in C for tokenization](https://github.com/github/linguist/pull/3846). Enry still uses [extract_token](https://github.com/github/linguist/pull/3846/files#diff-d5179df0b71620e3fac4535cd1368d15L60) regex-based algorithm. See [#193](https://github.com/src-d/enry/issues/193).
* Bayesian classifier can't distinguish "SQL" from "PLpgSQL. See [#194](https://github.com/src-d/enry/issues/194).
* Detection of [generated files](https://github.com/github/linguist/blob/bf95666fc15e49d556f2def4d0a85338423c25f3/lib/linguist/generated.rb#L53) is not supported yet.
(Thus they are not excluded from CLI output). See [#213](https://github.com/src-d/enry/issues/213).
* XML detection strategy is not implemented. See [#192](https://github.com/src-d/enry/issues/192).
* Overriding languages and types though `.gitattributes` is not yet supported. See [#18](https://github.com/src-d/enry/issues/18).
* `enry` CLI output does NOT exclude `.gitignore`ed files and git submodules, as Linguist does
In all the cases above that have an issue number - we plan to update enry to match Linguist behavior.
Benchmarks
------------
Enry's language detection has been compared with Linguist's on [*linguist/samples*](https://github.com/github/linguist/tree/master/samples).
We got these results:
![histogram](benchmarks/histogram/distribution.png)
The histogram shows the _number of files_ (y-axis) per _time interval bucket_ (x-axis).
Most of the files were detected faster by enry.
There are several cases where enry is slower than Linguist due to
Go regexp engine being slower than Ruby's on, wich is based on [oniguruma](https://github.com/kkos/oniguruma) library, written in C.
See [instructions](#misc) for running enry with oniguruma.
Why Enry?
------------
In the movie [My Fair Lady](https://en.wikipedia.org/wiki/My_Fair_Lady), [Professor Henry Higgins](http://www.imdb.com/character/ch0011719/) is a linguist who at the very beginning of the movie enjoys guessing the origin of people based on their accent.
"Enry Iggins" is how [Eliza Doolittle](http://www.imdb.com/character/ch0011720/), [pronounces](https://www.youtube.com/watch?v=pwNKyTktDIE) the name of the Professor.
## Development
To run the tests use:
go test ./...
Setting `ENRY_TEST_REPO` to the path to existing checkout of Linguist will avoid cloning it and sepeed tests up.
Setting `ENRY_DEBUG=1` will provide insight in the Bayesian classifier building done by `make code-generate`.
### Sync with github/linguist upstream
*enry* re-uses parts of the original [github/linguist](https://github.com/github/linguist) to generate internal data structures.
In order to update to the latest release of linguist do:
```bash
$ git clone https://github.com/github/linguist.git .linguist
$ cd .linguist; git checkout <release-tag>; cd ..
# put the new release's commit sha in the generator_test.go (to re-generate .gold test fixtures)
# https://github.com/go-enry/go-enry/blob/13d3d66d37a87f23a013246a1b0678c9ee3d524b/internal/code-generator/generator/generator_test.go#L18
$ make code-generate
```
To stay in sync, enry needs to be updated when a new release of the linguist includes changes to any of the following files:
* [languages.yml](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml)
* [heuristics.yml](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.yml)
* [vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml)
* [documentation.yml](https://github.com/github/linguist/blob/master/lib/linguist/documentation.yml)
There is no automation for detecting the changes in the linguist project, so this process above has to be done manually from time to time.
When submitting a pull request syncing up to a new release, please make sure it only contains the changes in
the generated files (in [data](https://github.com/go-enry/go-enry/blob/master/data) subdirectory).
Separating all the necessary "manual" code changes to a different PR that includes some background description and an update to the documentation on ["divergences from linguist"](#divergences-from-linguist) is very much appreciated as it simplifies the maintenance (review/release notes/etc).
## Misc
<details>
<summary>Running a benchmark & faster regexp engine</summary>
### Benchmark
All benchmark scripts are in [*benchmarks*](https://github.com/go-enry/go-enry/blob/master/benchmarks) directory.
#### Dependencies
As benchmarks depend on Ruby and Github-Linguist gem make sure you have:
- Ruby (e.g using [`rbenv`](https://github.com/rbenv/rbenv)), [`bundler`](https://bundler.io/) installed
- Docker
- [native dependencies](https://github.com/github/linguist/#dependencies) installed
- Build the gem `cd .linguist && bundle install && rake build_gem && cd -`
- Install it `gem install --no-rdoc --no-ri --local .linguist/github-linguist-*.gem`
#### Quick benchmark
To run quicker benchmarks
make benchmarks
to get average times for the primary detection function and strategies for the whole samples set. If you want to see measures per sample file use:
make benchmarks-samples
#### Full benchmark
If you want to reproduce the same benchmarks as reported above:
- Make sure all [dependencies](#benchmark-dependencies) are installed
- Install [gnuplot](http://gnuplot.info) (in order to plot the histogram)
- Run `ENRY_TEST_REPO="$PWD/.linguist" benchmarks/run.sh` (takes ~15h)
It will run the benchmarks for enry and Linguist, parse the output, create csv files and plot the histogram.
### Faster regexp engine (optional)
[Oniguruma](https://github.com/kkos/oniguruma) is CRuby's regular expression engine.
It is very fast and performs better than the one built into Go runtime. *enry* supports swapping
between those two engines thanks to [rubex](https://github.com/moovweb/rubex) project.
The typical overall speedup from using Oniguruma is 1.5-2x. However, it requires CGo and the external shared library.
On macOS with [Homebrew](https://brew.sh/), it is:
```
brew install oniguruma
```
On Ubuntu, it is
```
sudo apt install libonig-dev
```
To build enry with Oniguruma regexps use the `oniguruma` build tag
```
go get -v -t --tags oniguruma ./...
```
and then rebuild the project.
</details>
License
------------
Apache License, Version 2.0. See [LICENSE](LICENSE)

@ -4,16 +4,16 @@ import (
"math"
"sort"
"github.com/src-d/enry/v2/internal/tokenizer"
"github.com/go-enry/go-enry/v2/internal/tokenizer"
)
// Classifier is the interface in charge to detect the possible languages of the given content based on a set of
// classifier is the interface in charge to detect the possible languages of the given content based on a set of
// candidates. Candidates is a map which can be used to assign weights to languages dynamically.
type Classifier interface {
Classify(content []byte, candidates map[string]float64) (languages []string)
type classifier interface {
classify(content []byte, candidates map[string]float64) (languages []string)
}
type classifier struct {
type naiveBayes struct {
languagesLogProbabilities map[string]float64
tokensLogProbabilities map[string]map[string]float64
tokensTotal float64
@ -24,8 +24,8 @@ type scoredLanguage struct {
score float64
}
// Classify returns a sorted slice of possible languages sorted by decreasing language's probability
func (c *classifier) Classify(content []byte, candidates map[string]float64) []string {
// classify returns a sorted slice of possible languages sorted by decreasing language's probability
func (c *naiveBayes) classify(content []byte, candidates map[string]float64) []string {
var languages map[string]float64
if len(candidates) == 0 {
@ -73,7 +73,7 @@ func sortLanguagesByScore(scoredLangs []*scoredLanguage) []string {
return sortedLanguages
}
func (c *classifier) knownLangs() map[string]float64 {
func (c *naiveBayes) knownLangs() map[string]float64 {
langs := make(map[string]float64, len(c.languagesLogProbabilities))
for lang := range c.languagesLogProbabilities {
langs[lang]++
@ -82,7 +82,7 @@ func (c *classifier) knownLangs() map[string]float64 {
return langs
}
func (c *classifier) tokensLogProbability(tokens []string, language string) float64 {
func (c *naiveBayes) tokensLogProbability(tokens []string, language string) float64 {
var sum float64
for _, token := range tokens {
sum += c.tokenProbability(token, language)
@ -91,7 +91,7 @@ func (c *classifier) tokensLogProbability(tokens []string, language string) floa
return sum
}
func (c *classifier) tokenProbability(token, language string) float64 {
func (c *naiveBayes) tokenProbability(token, language string) float64 {
tokenProb, ok := c.tokensLogProbabilities[language][token]
if !ok {
tokenProb = math.Log(1.000000 / c.tokensTotal)

@ -6,8 +6,8 @@ import (
"path/filepath"
"strings"
"github.com/src-d/enry/v2/data"
"github.com/src-d/enry/v2/regex"
"github.com/go-enry/go-enry/v2/data"
"github.com/go-enry/go-enry/v2/regex"
)
// OtherLanguage is used as a zero value when a function can not return a specific language.
@ -26,8 +26,8 @@ var DefaultStrategies = []Strategy{
GetLanguagesByClassifier,
}
// DefaultClassifier is a Naive Bayes classifier trained on Linguist samples.
var DefaultClassifier Classifier = &classifier{
// defaultClassifier is a Naive Bayes classifier trained on Linguist samples.
var defaultClassifier classifier = &naiveBayes{
languagesLogProbabilities: data.LanguagesLogProbabilities,
tokensLogProbabilities: data.TokensLogProbabilities,
tokensTotal: data.TokensTotal,
@ -92,7 +92,7 @@ func GetLanguageByContent(filename string, content []byte) (language string, saf
}
// GetLanguageByClassifier returns the most probably language detected for the given content. It uses
// DefaultClassifier, if no candidates are provided it returns OtherLanguage.
// defaultClassifier, if no candidates are provided it returns OtherLanguage.
func GetLanguageByClassifier(content []byte, candidates []string) (language string, safe bool) {
return getLanguageByStrategy(GetLanguagesByClassifier, "", content, candidates)
}
@ -108,10 +108,10 @@ func getFirstLanguageAndSafe(languages []string) (language string, safe bool) {
return
}
// GetLanguageBySpecificClassifier returns the most probably language for the given content using
// getLanguageBySpecificClassifier returns the most probably language for the given content using
// classifier to detect language.
func GetLanguageBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (language string, safe bool) {
languages := GetLanguagesBySpecificClassifier(content, candidates, classifier)
func getLanguageBySpecificClassifier(content []byte, candidates []string, classifier classifier) (language string, safe bool) {
languages := getLanguagesBySpecificClassifier(content, candidates, classifier)
return getFirstLanguageAndSafe(languages)
}
@ -413,27 +413,28 @@ func GetLanguagesByContent(filename string, content []byte, _ []string) []string
return heuristic.Match(content)
}
// GetLanguagesByClassifier uses DefaultClassifier as a Classifier and returns a sorted slice of possible languages ordered by
// decreasing language's probability. If there are not candidates it returns nil. It complies with the signature to be a Strategy type.
// GetLanguagesByClassifier returns a sorted slice of possible languages ordered by
// decreasing language's probability. If there are not candidates it returns nil.
// It is a Strategy that uses a pre-trained defaultClassifier.
func GetLanguagesByClassifier(filename string, content []byte, candidates []string) (languages []string) {
if len(candidates) == 0 {
return nil
}
return GetLanguagesBySpecificClassifier(content, candidates, DefaultClassifier)
return getLanguagesBySpecificClassifier(content, candidates, defaultClassifier)
}
// GetLanguagesBySpecificClassifier returns a slice of possible languages. It takes in a Classifier to be used.
func GetLanguagesBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (languages []string) {
// getLanguagesBySpecificClassifier returns a slice of possible languages. It takes in a Classifier to be used.
func getLanguagesBySpecificClassifier(content []byte, candidates []string, classifier classifier) (languages []string) {
mapCandidates := make(map[string]float64)
for _, candidate := range candidates {
mapCandidates[candidate]++
}
return classifier.Classify(content, mapCandidates)
return classifier.classify(content, mapCandidates)
}
// GetLanguageExtensions returns the different extensions being used by the language.
// GetLanguageExtensions returns all extensions associated with the given language.
func GetLanguageExtensions(language string) []string {
return data.ExtensionsByLanguage[language]
}
@ -470,3 +471,12 @@ func GetLanguageByAlias(alias string) (lang string, ok bool) {
return
}
// GetLanguageGroup returns language group or empty string if language does not have group.
func GetLanguageGroup(language string) string {
if group, ok := data.LanguagesGroup[language]; ok {
return group
}
return ""
}

@ -0,0 +1,842 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
import "strings"
// LanguageByAliasMap keeps alias for different languages and use the name of the languages as an alias too.
// All the keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores.
var LanguageByAliasMap = map[string]string{
"1c_enterprise": "1C Enterprise",
"4d": "4D",
"abap": "ABAP",
"abl": "OpenEdge ABL",
"abnf": "ABNF",
"abuild": "Alpine Abuild",
"acfm": "Adobe Font Metrics",
"aconf": "ApacheConf",
"actionscript": "ActionScript",
"actionscript3": "ActionScript",
"actionscript_3": "ActionScript",
"ada": "Ada",
"ada2005": "Ada",
"ada95": "Ada",
"adobe_composite_font_metrics": "Adobe Font Metrics",
"adobe_font_metrics": "Adobe Font Metrics",
"adobe_multiple_font_metrics": "Adobe Font Metrics",
"advpl": "xBase",
"afdko": "OpenType Feature File",
"agda": "Agda",
"ags": "AGS Script",
"ags_script": "AGS Script",
"ahk": "AutoHotkey",
"alloy": "Alloy",
"alpine_abuild": "Alpine Abuild",
"altium": "Altium Designer",
"altium_designer": "Altium Designer",
"amfm": "Adobe Font Metrics",
"ampl": "AMPL",
"amusewiki": "Muse",
"angelscript": "AngelScript",
"ant_build_system": "Ant Build System",
"antlr": "ANTLR",
"apache": "ApacheConf",
"apacheconf": "ApacheConf",
"apex": "Apex",
"api_blueprint": "API Blueprint",
"apkbuild": "Alpine Abuild",
"apl": "APL",
"apollo_guidance_computer": "Apollo Guidance Computer",
"applescript": "AppleScript",
"arc": "Arc",
"arexx": "REXX",
"as3": "ActionScript",
"asciidoc": "AsciiDoc",
"asm": "Assembly",
"asn.1": "ASN.1",
"asp": "ASP",
"aspectj": "AspectJ",
"aspx": "ASP",
"aspx-vb": "ASP",
"assembly": "Assembly",
"asymptote": "Asymptote",
"ats": "ATS",
"ats2": "ATS",
"au3": "AutoIt",
"augeas": "Augeas",
"autoconf": "M4Sugar",
"autohotkey": "AutoHotkey",
"autoit": "AutoIt",
"autoit3": "AutoIt",
"autoitscript": "AutoIt",
"awk": "Awk",
"b3d": "BlitzBasic",
"ballerina": "Ballerina",
"bash": "Shell",
"bash_session": "ShellSession",
"bat": "Batchfile",
"batch": "Batchfile",
"batchfile": "Batchfile",
"bazel": "Starlark",
"befunge": "Befunge",
"bibtex": "BibTeX",
"bison": "Bison",
"bitbake": "BitBake",
"blade": "Blade",
"blitz3d": "BlitzBasic",
"blitzbasic": "BlitzBasic",
"blitzmax": "BlitzMax",
"blitzplus": "BlitzBasic",
"bluespec": "Bluespec",
"bmax": "BlitzMax",
"boo": "Boo",
"bplus": "BlitzBasic",
"brainfuck": "Brainfuck",
"brightscript": "Brightscript",
"bro": "Zeek",
"bsdmake": "Makefile",
"byond": "DM",
"bzl": "Starlark",
"c": "C",
"c#": "C#",
"c++": "C++",
"c++-objdump": "Cpp-ObjDump",
"c-objdump": "C-ObjDump",
"c2hs": "C2hs Haskell",
"c2hs_haskell": "C2hs Haskell",
"cabal": "Cabal Config",
"cabal_config": "Cabal Config",
"cap'n_proto": "Cap'n Proto",
"carto": "CartoCSS",
"cartocss": "CartoCSS",
"ceylon": "Ceylon",
"cfc": "ColdFusion CFC",
"cfm": "ColdFusion",
"cfml": "ColdFusion",
"chapel": "Chapel",
"charity": "Charity",
"chpl": "Chapel",
"chuck": "ChucK",
"cirru": "Cirru",
"clarion": "Clarion",
"clean": "Clean",
"click": "Click",
"clipper": "xBase",
"clips": "CLIPS",
"clojure": "Clojure",
"closure_templates": "Closure Templates",
"cloud_firestore_security_rules": "Cloud Firestore Security Rules",
"cmake": "CMake",
"cobol": "COBOL",
"coccinelle": "SmPL",
"codeql": "CodeQL",
"coffee": "CoffeeScript",
"coffee-script": "CoffeeScript",
"coffeescript": "CoffeeScript",
"coldfusion": "ColdFusion",
"coldfusion_cfc": "ColdFusion CFC",
"coldfusion_html": "ColdFusion",
"collada": "COLLADA",
"common_lisp": "Common Lisp",
"common_workflow_language": "Common Workflow Language",
"component_pascal": "Component Pascal",
"conll": "CoNLL-U",
"conll-u": "CoNLL-U",
"conll-x": "CoNLL-U",
"console": "ShellSession",
"cool": "Cool",
"coq": "Coq",
"cperl": "Perl",
"cpp": "C++",
"cpp-objdump": "Cpp-ObjDump",
"creole": "Creole",
"crystal": "Crystal",
"csharp": "C#",
"cson": "CSON",
"csound": "Csound",
"csound-csd": "Csound Document",
"csound-orc": "Csound",
"csound-sco": "Csound Score",
"csound_document": "Csound Document",
"csound_score": "Csound Score",
"css": "CSS",
"csv": "CSV",
"cucumber": "Gherkin",
"cuda": "Cuda",
"curl_config": "cURL Config",
"curlrc": "cURL Config",
"cweb": "CWeb",
"cwl": "Common Workflow Language",
"cycript": "Cycript",
"cython": "Cython",
"d": "D",
"d-objdump": "D-ObjDump",
"darcs_patch": "Darcs Patch",
"dart": "Dart",
"dataweave": "DataWeave",
"dcl": "DIGITAL Command Language",
"delphi": "Component Pascal",
"desktop": "desktop",
"dhall": "Dhall",
"diff": "Diff",
"digital_command_language": "DIGITAL Command Language",
"dircolors": "dircolors",
"directx_3d_file": "DirectX 3D File",
"django": "HTML+Django",
"dm": "DM",
"dns_zone": "DNS Zone",
"dockerfile": "Dockerfile",
"dogescript": "Dogescript",
"dosbatch": "Batchfile",
"dosini": "INI",
"dpatch": "Darcs Patch",
"dtrace": "DTrace",
"dtrace-script": "DTrace",
"dylan": "Dylan",
"e": "E",
"eagle": "Eagle",
"easybuild": "Easybuild",
"ebnf": "EBNF",
"ec": "eC",
"ecere_projects": "Ecere Projects",
"ecl": "ECL",
"eclipse": "ECLiPSe",
"ecr": "HTML+ECR",
"editor-config": "EditorConfig",
"editorconfig": "EditorConfig",
"edje_data_collection": "Edje Data Collection",
"edn": "edn",
"eeschema_schematic": "KiCad Schematic",
"eex": "HTML+EEX",
"eiffel": "Eiffel",
"ejs": "EJS",
"elisp": "Emacs Lisp",
"elixir": "Elixir",
"elm": "Elm",
"emacs": "Emacs Lisp",
"emacs_lisp": "Emacs Lisp",
"emacs_muse": "Muse",
"emberscript": "EmberScript",
"eml": "EML",
"eq": "EQ",
"erb": "HTML+ERB",
"erlang": "Erlang",
"f#": "F#",
"f*": "F*",
"factor": "Factor",
"fancy": "Fancy",
"fantom": "Fantom",
"faust": "Faust",
"figfont": "FIGlet Font",
"figlet_font": "FIGlet Font",
"filebench_wml": "Filebench WML",
"filterscript": "Filterscript",
"fish": "fish",
"flex": "Lex",
"flux": "FLUX",
"formatted": "Formatted",
"forth": "Forth",
"fortran": "Fortran",
"foxpro": "xBase",
"freemarker": "FreeMarker",
"frege": "Frege",
"fsharp": "F#",
"fstar": "F*",
"ftl": "FreeMarker",
"fundamental": "Text",
"g-code": "G-code",
"game_maker_language": "Game Maker Language",
"gaml": "GAML",
"gams": "GAMS",
"gap": "GAP",
"gcc_machine_description": "GCC Machine Description",
"gdb": "GDB",
"gdscript": "GDScript",
"genie": "Genie",
"genshi": "Genshi",
"gentoo_ebuild": "Gentoo Ebuild",
"gentoo_eclass": "Gentoo Eclass",
"gerber_image": "Gerber Image",
"gettext_catalog": "Gettext Catalog",
"gf": "Grammatical Framework",
"gherkin": "Gherkin",
"git-ignore": "Ignore List",
"git_attributes": "Git Attributes",
"git_config": "Git Config",
"gitattributes": "Git Attributes",
"gitconfig": "Git Config",
"gitignore": "Ignore List",
"gitmodules": "Git Config",
"glsl": "GLSL",
"glyph": "Glyph",
"glyph_bitmap_distribution_format": "Glyph Bitmap Distribution Format",
"gn": "GN",
"gnuplot": "Gnuplot",
"go": "Go",
"golang": "Go",
"golo": "Golo",
"gosu": "Gosu",
"grace": "Grace",
"gradle": "Gradle",
"grammatical_framework": "Grammatical Framework",
"graph_modeling_language": "Graph Modeling Language",
"graphql": "GraphQL",
"graphviz_(dot)": "Graphviz (DOT)",
"groff": "Roff",
"groovy": "Groovy",
"groovy_server_pages": "Groovy Server Pages",
"gsp": "Groovy Server Pages",
"hack": "Hack",
"haml": "Haml",
"handlebars": "Handlebars",
"haproxy": "HAProxy",
"harbour": "Harbour",
"haskell": "Haskell",
"haxe": "Haxe",
"hbs": "Handlebars",
"hcl": "HCL",
"hiveql": "HiveQL",
"hlsl": "HLSL",
"holyc": "HolyC",
"html": "HTML",
"html+django": "HTML+Django",
"html+django/jinja": "HTML+Django",
"html+ecr": "HTML+ECR",
"html+eex": "HTML+EEX",
"html+erb": "HTML+ERB",
"html+jinja": "HTML+Django",
"html+php": "HTML+PHP",
"html+razor": "HTML+Razor",
"html+ruby": "RHTML",
"htmlbars": "Handlebars",
"htmldjango": "HTML+Django",
"http": "HTTP",
"hxml": "HXML",
"hy": "Hy",
"hylang": "Hy",
"hyphy": "HyPhy",
"i7": "Inform 7",
"idl": "IDL",
"idris": "Idris",
"ignore": "Ignore List",
"ignore_list": "Ignore List",
"igor": "IGOR Pro",
"igor_pro": "IGOR Pro",
"igorpro": "IGOR Pro",
"inc": "PHP",
"inform7": "Inform 7",
"inform_7": "Inform 7",
"ini": "INI",
"inno_setup": "Inno Setup",
"inputrc": "Readline Config",
"io": "Io",
"ioke": "Ioke",
"ipython_notebook": "Jupyter Notebook",
"irc": "IRC log",
"irc_log": "IRC log",
"irc_logs": "IRC log",
"isabelle": "Isabelle",
"isabelle_root": "Isabelle ROOT",
"j": "J",
"jasmin": "Jasmin",
"java": "Java",
"java_properties": "Java Properties",
"java_server_page": "Groovy Server Pages",
"java_server_pages": "Java Server Pages",
"javascript": "JavaScript",
"javascript+erb": "JavaScript+ERB",
"jflex": "JFlex",
"jison": "Jison",
"jison_lex": "Jison Lex",
"jolie": "Jolie",
"jruby": "Ruby",
"js": "JavaScript",
"json": "JSON",
"json5": "JSON5",
"json_with_comments": "JSON with Comments",
"jsonc": "JSON with Comments",
"jsoniq": "JSONiq",
"jsonld": "JSONLD",
"jsonnet": "Jsonnet",
"jsp": "Java Server Pages",
"jsx": "JSX",
"julia": "Julia",
"jupyter_notebook": "Jupyter Notebook",
"kicad_layout": "KiCad Layout",
"kicad_legacy_layout": "KiCad Legacy Layout",
"kicad_schematic": "KiCad Schematic",
"kit": "Kit",
"kotlin": "Kotlin",
"krl": "KRL",
"labview": "LabVIEW",
"lasso": "Lasso",
"lassoscript": "Lasso",
"latex": "TeX",
"latte": "Latte",
"lean": "Lean",
"less": "Less",
"lex": "Lex",
"lfe": "LFE",
"lhaskell": "Literate Haskell",
"lhs": "Literate Haskell",
"lilypond": "LilyPond",
"limbo": "Limbo",
"linker_script": "Linker Script",
"linux_kernel_module": "Linux Kernel Module",
"liquid": "Liquid",
"lisp": "Common Lisp",
"litcoffee": "Literate CoffeeScript",
"literate_agda": "Literate Agda",
"literate_coffeescript": "Literate CoffeeScript",
"literate_haskell": "Literate Haskell",
"live-script": "LiveScript",
"livescript": "LiveScript",
"llvm": "LLVM",
"logos": "Logos",
"logtalk": "Logtalk",
"lolcode": "LOLCODE",
"lookml": "LookML",
"loomscript": "LoomScript",
"ls": "LiveScript",
"lsl": "LSL",
"ltspice_symbol": "LTspice Symbol",
"lua": "Lua",
"m": "M",
"m4": "M4",
"m4sugar": "M4Sugar",
"m68k": "Motorola 68K Assembly",
"macruby": "Ruby",
"make": "Makefile",
"makefile": "Makefile",
"mako": "Mako",
"man": "Roff",
"man-page": "Roff",
"man_page": "Roff",
"manpage": "Roff",
"markdown": "Markdown",
"marko": "Marko",
"markojs": "Marko",
"mask": "Mask",
"mathematica": "Mathematica",
"matlab": "MATLAB",
"maven_pom": "Maven POM",
"max": "Max",
"max/msp": "Max",
"maxmsp": "Max",
"maxscript": "MAXScript",
"mcfunction": "mcfunction",
"mdoc": "Roff",
"mediawiki": "MediaWiki",
"mercury": "Mercury",
"meson": "Meson",
"metal": "Metal",
"mf": "Makefile",
"microsoft_developer_studio_project": "Microsoft Developer Studio Project",
"minid": "MiniD",
"mirah": "Mirah",
"mirc_script": "mIRC Script",
"mlir": "MLIR",
"mma": "Mathematica",
"modelica": "Modelica",
"modula-2": "Modula-2",
"modula-3": "Modula-3",
"module_management_system": "Module Management System",
"monkey": "Monkey",
"moocode": "Moocode",
"moonscript": "MoonScript",
"motorola_68k_assembly": "Motorola 68K Assembly",
"mql4": "MQL4",
"mql5": "MQL5",
"mtml": "MTML",
"muf": "MUF",
"mumps": "M",
"mupad": "mupad",
"muse": "Muse",
"myghty": "Myghty",
"nanorc": "nanorc",
"nasl": "NASL",
"nasm": "Assembly",
"ncl": "NCL",
"nearley": "Nearley",
"nemerle": "Nemerle",
"neosnippet": "Vim Snippet",
"nesc": "nesC",
"netlinx": "NetLinx",
"netlinx+erb": "NetLinx+ERB",
"netlogo": "NetLogo",
"newlisp": "NewLisp",
"nextflow": "Nextflow",
"nginx": "Nginx",
"nginx_configuration_file": "Nginx",
"nim": "Nim",
"ninja": "Ninja",
"nit": "Nit",
"nix": "Nix",
"nixos": "Nix",
"njk": "HTML+Django",
"nl": "NL",
"node": "JavaScript",
"npm_config": "NPM Config",
"npmrc": "NPM Config",
"nroff": "Roff",
"nsis": "NSIS",
"nu": "Nu",
"numpy": "NumPy",
"nunjucks": "HTML+Django",
"nush": "Nu",
"nvim": "Vim script",
"obj-c": "Objective-C",
"obj-c++": "Objective-C++",
"obj-j": "Objective-J",
"objc": "Objective-C",
"objc++": "Objective-C++",
"objdump": "ObjDump",
"object_data_instance_notation": "Object Data Instance Notation",
"objective-c": "Objective-C",
"objective-c++": "Objective-C++",
"objective-j": "Objective-J",
"objectivec": "Objective-C",
"objectivec++": "Objective-C++",
"objectivej": "Objective-J",
"objectpascal": "Component Pascal",
"objectscript": "ObjectScript",
"objj": "Objective-J",
"ocaml": "OCaml",
"octave": "MATLAB",
"odin": "Odin",
"odin-lang": "Odin",
"odinlang": "Odin",
"omgrofl": "Omgrofl",
"oncrpc": "RPC",
"ooc": "ooc",
"opa": "Opa",
"opal": "Opal",
"open_policy_agent": "Open Policy Agent",
"opencl": "OpenCL",
"openedge": "OpenEdge ABL",
"openedge_abl": "OpenEdge ABL",
"openqasm": "OpenQASM",
"openrc": "OpenRC runscript",
"openrc_runscript": "OpenRC runscript",
"openscad": "OpenSCAD",
"openstep_property_list": "OpenStep Property List",
"opentype_feature_file": "OpenType Feature File",
"org": "Org",
"osascript": "AppleScript",
"ox": "Ox",
"oxygene": "Oxygene",
"oz": "Oz",
"p4": "P4",
"pan": "Pan",
"pandoc": "Markdown",
"papyrus": "Papyrus",
"parrot": "Parrot",
"parrot_assembly": "Parrot Assembly",
"parrot_internal_representation": "Parrot Internal Representation",
"pascal": "Pascal",
"pasm": "Parrot Assembly",
"pawn": "Pawn",
"pcbnew": "KiCad Layout",
"pep8": "Pep8",
"perl": "Perl",
"perl-6": "Raku",
"perl6": "Raku",
"php": "PHP",
"pic": "Pic",
"pickle": "Pickle",
"picolisp": "PicoLisp",
"piglatin": "PigLatin",
"pike": "Pike",
"pir": "Parrot Internal Representation",
"plantuml": "PlantUML",
"plpgsql": "PLpgSQL",
"plsql": "PLSQL",
"pod": "Pod",
"pod_6": "Pod 6",
"pogoscript": "PogoScript",
"pony": "Pony",
"posh": "PowerShell",
"postcss": "PostCSS",
"postscr": "PostScript",
"postscript": "PostScript",
"pot": "Gettext Catalog",
"pov-ray": "POV-Ray SDL",
"pov-ray_sdl": "POV-Ray SDL",
"povray": "POV-Ray SDL",
"powerbuilder": "PowerBuilder",
"powershell": "PowerShell",
"prisma": "Prisma",
"processing": "Processing",
"progress": "OpenEdge ABL",
"proguard": "Proguard",
"prolog": "Prolog",
"propeller_spin": "Propeller Spin",
"protobuf": "Protocol Buffer",
"protocol_buffer": "Protocol Buffer",
"protocol_buffers": "Protocol Buffer",
"public_key": "Public Key",
"pug": "Pug",
"puppet": "Puppet",
"pure_data": "Pure Data",
"purebasic": "PureBasic",
"purescript": "PureScript",
"pwsh": "PowerShell",
"pycon": "Python console",
"pyrex": "Cython",
"python": "Python",
"python3": "Python",
"python_console": "Python console",
"python_traceback": "Python traceback",
"q": "q",
"ql": "CodeQL",
"qmake": "QMake",
"qml": "QML",
"quake": "Quake",
"r": "R",
"racket": "Racket",
"ragel": "Ragel",
"ragel-rb": "Ragel",
"ragel-ruby": "Ragel",
"rake": "Ruby",
"raku": "Raku",
"raml": "RAML",
"rascal": "Rascal",
"raw": "Raw token data",
"raw_token_data": "Raw token data",
"razor": "HTML+Razor",
"rb": "Ruby",
"rbx": "Ruby",
"rdoc": "RDoc",
"readline": "Readline Config",
"readline_config": "Readline Config",
"realbasic": "REALbasic",
"reason": "Reason",
"rebol": "Rebol",
"red": "Red",
"red/system": "Red",
"redcode": "Redcode",
"regex": "Regular Expression",
"regexp": "Regular Expression",
"regular_expression": "Regular Expression",
"ren'py": "Ren'Py",
"renderscript": "RenderScript",
"renpy": "Ren'Py",
"restructuredtext": "reStructuredText",
"rexx": "REXX",
"rhtml": "RHTML",
"rich_text_format": "Rich Text Format",
"ring": "Ring",
"riot": "Riot",
"rmarkdown": "RMarkdown",
"robotframework": "RobotFramework",
"roff": "Roff",
"roff_manpage": "Roff Manpage",
"rouge": "Rouge",
"rpc": "RPC",
"rpcgen": "RPC",
"rpm_spec": "RPM Spec",
"rs-274x": "Gerber Image",
"rscript": "R",
"rss": "XML",
"rst": "reStructuredText",
"ruby": "Ruby",
"runoff": "RUNOFF",
"rust": "Rust",
"rusthon": "Python",
"sage": "Sage",
"salt": "SaltStack",
"saltstack": "SaltStack",
"saltstate": "SaltStack",
"sas": "SAS",
"sass": "Sass",
"scala": "Scala",
"scaml": "Scaml",
"scheme": "Scheme",
"scilab": "Scilab",
"scss": "SCSS",
"sed": "sed",
"self": "Self",
"sh": "Shell",
"shaderlab": "ShaderLab",
"shell": "Shell",
"shell-script": "Shell",
"shellsession": "ShellSession",
"shen": "Shen",
"slash": "Slash",
"slice": "Slice",
"slim": "Slim",
"smali": "Smali",
"smalltalk": "Smalltalk",
"smarty": "Smarty",
"sml": "Standard ML",
"smpl": "SmPL",
"smt": "SMT",
"snipmate": "Vim Snippet",
"snippet": "YASnippet",
"solidity": "Solidity",
"sourcemod": "SourcePawn",
"sourcepawn": "SourcePawn",
"soy": "Closure Templates",
"sparql": "SPARQL",
"specfile": "RPM Spec",
"spline_font_database": "Spline Font Database",
"splus": "R",
"sqf": "SQF",
"sql": "SQL",
"sqlpl": "SQLPL",
"squeak": "Smalltalk",
"squirrel": "Squirrel",
"srecode_template": "SRecode Template",
"ssh_config": "SSH Config",
"stan": "Stan",
"standard_ml": "Standard ML",
"starlark": "Starlark",
"stata": "Stata",
"ston": "STON",
"stylus": "Stylus",
"subrip_text": "SubRip Text",
"sugarss": "SugarSS",
"supercollider": "SuperCollider",
"svelte": "Svelte",
"svg": "SVG",
"swift": "Swift",
"swig": "SWIG",
"systemverilog": "SystemVerilog",
"tcl": "Tcl",
"tcsh": "Tcsh",
"tea": "Tea",
"terra": "Terra",
"terraform": "HCL",
"tex": "TeX",
"texinfo": "Texinfo",
"text": "Text",
"textile": "Textile",
"thrift": "Thrift",
"ti_program": "TI Program",
"tl": "Type Language",
"tla": "TLA",
"toml": "TOML",
"troff": "Roff",
"ts": "TypeScript",
"tsql": "TSQL",
"tsx": "TSX",
"turing": "Turing",
"turtle": "Turtle",
"twig": "Twig",
"txl": "TXL",
"type_language": "Type Language",
"typescript": "TypeScript",
"udiff": "Diff",
"ultisnip": "Vim Snippet",
"ultisnips": "Vim Snippet",
"unified_parallel_c": "Unified Parallel C",
"unity3d_asset": "Unity3D Asset",
"unix_assembly": "Unix Assembly",
"uno": "Uno",
"unrealscript": "UnrealScript",
"ur": "UrWeb",
"ur/web": "UrWeb",
"urweb": "UrWeb",
"v": "V",
"vala": "Vala",
"vb.net": "Visual Basic .NET",
"vb6": "VBA",
"vb_.net": "Visual Basic .NET",
"vba": "VBA",
"vbnet": "Visual Basic .NET",
"vbscript": "VBScript",
"vcl": "VCL",
"verilog": "Verilog",
"vhdl": "VHDL",
"vim": "Vim script",
"vim_script": "Vim script",
"vim_snippet": "Vim Snippet",
"viml": "Vim script",
"visual_basic": "Visual Basic .NET",
"visual_basic_.net": "Visual Basic .NET",
"visual_basic_6": "VBA",
"visual_basic_for_applications": "VBA",
"vlang": "V",
"volt": "Volt",
"vue": "Vue",
"wasm": "WebAssembly",
"wast": "WebAssembly",
"wavefront_material": "Wavefront Material",
"wavefront_object": "Wavefront Object",
"wdl": "wdl",
"web_ontology_language": "Web Ontology Language",
"webassembly": "WebAssembly",
"webidl": "WebIDL",
"webvtt": "WebVTT",
"wget_config": "Wget Config",
"wgetrc": "Wget Config",
"winbatch": "Batchfile",
"windows_registry_entries": "Windows Registry Entries",
"wisp": "wisp",
"wollok": "Wollok",
"world_of_warcraft_addon_data": "World of Warcraft Addon Data",
"wsdl": "XML",
"x10": "X10",
"x_bitmap": "X BitMap",
"x_font_directory_index": "X Font Directory Index",
"x_pixmap": "X PixMap",
"xbase": "xBase",
"xbm": "X BitMap",
"xc": "XC",
"xcompose": "XCompose",
"xdr": "RPC",
"xhtml": "HTML",
"xml": "XML",
"xml+genshi": "Genshi",
"xml+kid": "Genshi",
"xml_property_list": "XML Property List",
"xojo": "Xojo",
"xpages": "XPages",
"xpm": "X PixMap",
"xproc": "XProc",
"xquery": "XQuery",
"xs": "XS",
"xsd": "XML",
"xsl": "XSLT",
"xslt": "XSLT",
"xten": "X10",
"xtend": "Xtend",
"yacc": "Yacc",
"yaml": "YAML",
"yang": "YANG",
"yara": "YARA",
"yas": "YASnippet",
"yasnippet": "YASnippet",
"yml": "YAML",
"zap": "ZAP",
"zeek": "Zeek",
"zenscript": "ZenScript",
"zephir": "Zephir",
"zig": "Zig",
"zil": "ZIL",
"zimpl": "Zimpl",
"zsh": "Shell",
}
// LanguageByAlias looks up the language name by it's alias or name.
// It mirrors the logic of github linguist and is needed e.g for heuristcs.yml
// that mixes names and aliases in a language field (see XPM example).
func LanguageByAlias(langOrAlias string) (lang string, ok bool) {
k := convertToAliasKey(langOrAlias)
lang, ok = LanguageByAliasMap[k]
return
}
// convertToAliasKey converts language name to a key in LanguageByAliasMap.
// Following
// - internal.code-generator.generator.convertToAliasKey()
// - GetLanguageByAlias()
// conventions.
// It is here to avoid dependency on "generate" and "enry" packages.
func convertToAliasKey(langName string) string {
ak := strings.SplitN(langName, `,`, 2)[0]
ak = strings.Replace(ak, ` `, `_`, -1)
ak = strings.ToLower(ak)
return ak
}

@ -1,5 +1,5 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
@ -71,11 +71,13 @@ var LanguagesColor = map[string]string{
"Factor": "#636746",
"Fancy": "#7b9db4",
"Fantom": "#14253c",
"Faust": "#c37240",
"Forth": "#341708",
"Fortran": "#4d41b1",
"FreeMarker": "#0050b2",
"Frege": "#00cafe",
"G-code": "#D08CF2",
"GAML": "#FFC766",
"GDScript": "#355570",
"Game Maker Language": "#71b417",
"Genie": "#fb855d",
@ -96,6 +98,7 @@ var LanguagesColor = map[string]string{
"HolyC": "#ffefaf",
"Hy": "#7790B2",
"IDL": "#a3522f",
"IGOR Pro": "#0000cc",
"Idris": "#b30000",
"Io": "#a9188d",
"Ioke": "#078193",
@ -121,6 +124,7 @@ var LanguagesColor = map[string]string{
"Lua": "#000080",
"MATLAB": "#e16737",
"MAXScript": "#00a6a6",
"MLIR": "#5EC8DB",
"MQL4": "#62A8D6",
"MQL5": "#4A76B8",
"MTML": "#b7e1f4",
@ -149,8 +153,10 @@ var LanguagesColor = map[string]string{
"Objective-C": "#438eff",
"Objective-C++": "#6866fb",
"Objective-J": "#ff0c5a",
"Odin": "#60AFFE",
"Omgrofl": "#cabbff",
"Opal": "#f7ede0",
"OpenQASM": "#AA70FF",
"Oxygene": "#cdd0e3",
"Oz": "#fab738",
"P4": "#7055b5",
@ -163,7 +169,6 @@ var LanguagesColor = map[string]string{
"Pawn": "#dbb284",
"Pep8": "#C76F5B",
"Perl": "#0298c3",
"Perl 6": "#0000fb",
"PigLatin": "#fcd7de",
"Pike": "#005390",
"PogoScript": "#d80074",
@ -184,11 +189,14 @@ var LanguagesColor = map[string]string{
"RUNOFF": "#665a4e",
"Racket": "#3c5caa",
"Ragel": "#9d5200",
"Raku": "#0000fb",
"Rascal": "#fffaa0",
"Reason": "#ff5847",
"Rebol": "#358a5b",
"Red": "#f50000",
"Ren'Py": "#ff7f7f",
"Ring": "#2D54CB",
"Riot": "#A71E49",
"Roff": "#ecdebe",
"Rouge": "#cc0088",
"Ruby": "#701516",
@ -204,12 +212,14 @@ var LanguagesColor = map[string]string{
"Shen": "#120F14",
"Slash": "#007eff",
"Slice": "#003fa2",
"SmPL": "#c94949",
"Smalltalk": "#596706",
"Solidity": "#AA6746",
"SourcePawn": "#5c7611",
"Squirrel": "#800000",
"Stan": "#b2011d",
"Standard ML": "#dc566d",
"Starlark": "#76d275",
"SuperCollider": "#46390b",
"Swift": "#ffac45",
"SystemVerilog": "#DAE1C2",
@ -220,12 +230,15 @@ var LanguagesColor = map[string]string{
"Turing": "#cf142b",
"TypeScript": "#2b7489",
"UnrealScript": "#a54c4d",
"V": "#5d87bd",
"VBA": "#867db1",
"VBScript": "#15dcdc",
"VCL": "#148AA8",
"VHDL": "#adb2cb",
"Vala": "#fbe5cd",
"Verilog": "#b2b7f8",
"Vim script": "#199f4b",
"Visual Basic": "#945db7",
"Visual Basic .NET": "#945db7",
"Volt": "#1F1F1F",
"Vue": "#2c3e50",
"WebAssembly": "#04133b",
@ -243,6 +256,7 @@ var LanguagesColor = map[string]string{
"Zephir": "#118f9e",
"Zig": "#ec915c",
"eC": "#913960",
"mIRC Script": "#926059",
"mcfunction": "#E22837",
"nesC": "#94B0C7",
"ooc": "#b0b77e",

@ -0,0 +1,7 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
// linguist's commit from which files were generated.
var LinguistCommit = "40992ba7f86889f80dfed3ba95e11e1082200bad"

@ -1,12 +1,12 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
import (
"regexp"
"github.com/src-d/enry/v2/data/rule"
"github.com/go-enry/go-enry/v2/data/rule"
)
var ContentHeuristics = map[string]*Heuristics{
@ -591,6 +591,12 @@ var ContentHeuristics = map[string]*Heuristics{
regexp.MustCompile(`(?m)^(\/\/.+|((import|export)\s+)?(function|int|float|char)\s+((room|repeatedly|on|game)_)?([A-Za-z]+[A-Za-z_0-9]+)\s*[;\(])`),
),
},
".asm": &Heuristics{
rule.Or(
rule.MatchingLanguages("Motorola 68K Assembly"),
regexp.MustCompile(`(?m)(?im)\bmoveq(?:\.l)?\s+#(?:\$-?[0-9a-f]{1,3}|%[0-1]{1,8}|-?[0-9]{1,3}),\s*d[0-7]\b|(?im)^\s*move(?:\.[bwl])?\s+(?:sr|usp),\s*[^\s]+|(?im)^\s*move\.[bwl]\s+.*\b[ad]\d|(?im)^\s*movem\.[bwl]\b|(?im)^\s*move[mp](?:\.[wl])?\b|(?im)^\s*btst\b|(?im)^\s*dbra\b`),
),
},
".asy": &Heuristics{
rule.Or(
rule.MatchingLanguages("LTspice Symbol"),
@ -673,6 +679,16 @@ var ContentHeuristics = map[string]*Heuristics{
regexp.MustCompile(`(?m)([\/\\].*:\s+.*\s\\$|: \\$|^[ %]:|^[\w\s\/\\.]+\w+\.\w+\s*:\s+[\w\s\/\\.]+\w+\.\w+)`),
),
},
".dsp": &Heuristics{
rule.Or(
rule.MatchingLanguages("Microsoft Developer Studio Project"),
regexp.MustCompile(`(?m)# Microsoft Developer Studio Generated Build File`),
),
rule.Or(
rule.MatchingLanguages("Faust"),
regexp.MustCompile(`(?m)\bprocess\s*[(=]|\b(library|import)\s*\(\s*"|\bdeclare\s+(name|version|author|copyright|license)\s+"`),
),
},
".ecl": &Heuristics{
rule.Or(
rule.MatchingLanguages("ECLiPSe"),
@ -772,6 +788,10 @@ var ContentHeuristics = map[string]*Heuristics{
),
},
".gs": &Heuristics{
rule.Or(
rule.MatchingLanguages("GLSL"),
regexp.MustCompile(`(?m)^#version\s+[0-9]+\b`),
),
rule.Or(
rule.MatchingLanguages("Gosu"),
regexp.MustCompile(`(?m)^uses java\.`),
@ -793,6 +813,16 @@ var ContentHeuristics = map[string]*Heuristics{
regexp.MustCompile(`(?m)<\?hh`),
),
},
".i": &Heuristics{
rule.Or(
rule.MatchingLanguages("Motorola 68K Assembly"),
regexp.MustCompile(`(?m)(?im)\bmoveq(?:\.l)?\s+#(?:\$-?[0-9a-f]{1,3}|%[0-1]{1,8}|-?[0-9]{1,3}),\s*d[0-7]\b|(?im)^\s*move(?:\.[bwl])?\s+(?:sr|usp),\s*[^\s]+|(?im)^\s*move\.[bwl]\s+.*\b[ad]\d|(?im)^\s*movem\.[bwl]\b|(?im)^\s*move[mp](?:\.[wl])?\b|(?im)^\s*btst\b|(?im)^\s*dbra\b`),
),
rule.Or(
rule.MatchingLanguages("SWIG"),
regexp.MustCompile(`(?m)^[ \t]*%[a-z_]+\b|^%[{}]$`),
),
},
".ice": &Heuristics{
rule.Or(
rule.MatchingLanguages("JSON"),
@ -803,6 +833,10 @@ var ContentHeuristics = map[string]*Heuristics{
),
},
".inc": &Heuristics{
rule.Or(
rule.MatchingLanguages("Motorola 68K Assembly"),
regexp.MustCompile(`(?m)(?im)\bmoveq(?:\.l)?\s+#(?:\$-?[0-9a-f]{1,3}|%[0-1]{1,8}|-?[0-9]{1,3}),\s*d[0-7]\b|(?im)^\s*move(?:\.[bwl])?\s+(?:sr|usp),\s*[^\s]+|(?im)^\s*move\.[bwl]\s+.*\b[ad]\d|(?im)^\s*movem\.[bwl]\b|(?im)^\s*move[mp](?:\.[wl])?\b|(?im)^\s*btst\b|(?im)^\s*dbra\b`),
),
rule.Or(
rule.MatchingLanguages("PHP"),
regexp.MustCompile(`(?m)^<\?(?:php)?`),
@ -931,6 +965,12 @@ var ContentHeuristics = map[string]*Heuristics{
rule.MatchingLanguages("Roff"),
),
},
".mask": &Heuristics{
rule.Or(
rule.MatchingLanguages("Unity3d Asset"),
regexp.MustCompile(`(?m)tag:unity3d.com`),
),
},
".md": &Heuristics{
rule.Or(
rule.MatchingLanguages("Markdown"),
@ -1047,6 +1087,25 @@ var ContentHeuristics = map[string]*Heuristics{
rule.MatchingLanguages("NewLisp"),
),
},
".odin": &Heuristics{
rule.Or(
rule.MatchingLanguages("Object Data Instance Notation"),
regexp.MustCompile(`(?m)(?:^|<)\s*[A-Za-z0-9_]+\s*=\s*<`),
),
rule.Or(
rule.MatchingLanguages("Odin"),
regexp.MustCompile(`(?m)package\s+\w+|\b(?:im|ex)port\s*"[\w:./]+"|\w+\s*::\s*(?:proc|struct)\s*\(|^\s*//\s`),
),
},
".p": &Heuristics{
rule.Or(
rule.MatchingLanguages("Gnuplot"),
regexp.MustCompile(`(?m)^s?plot\b|^set\s+(term|terminal|out|output|[xy]tics|[xy]label|[xy]range|style)\b`),
),
rule.Always(
rule.MatchingLanguages("OpenEdge ABL"),
),
},
".php": &Heuristics{
rule.Or(
rule.MatchingLanguages("Hack"),
@ -1067,17 +1126,26 @@ var ContentHeuristics = map[string]*Heuristics{
regexp.MustCompile(`(?m)\buse\s+(?:strict\b|v?5\.)`),
),
rule.Or(
rule.MatchingLanguages("Perl 6"),
rule.MatchingLanguages("Raku"),
regexp.MustCompile(`(?m)^\s*(?:use\s+v6\b|\bmodule\b|\b(?:my\s+)?class\b)`),
),
},
".plist": &Heuristics{
rule.Or(
rule.MatchingLanguages("XML Property List"),
regexp.MustCompile(`(?m)<!DOCTYPE\s+plist`),
),
rule.Always(
rule.MatchingLanguages("OpenStep Property List"),
),
},
".pm": &Heuristics{
rule.Or(
rule.MatchingLanguages("Perl"),
regexp.MustCompile(`(?m)\buse\s+(?:strict\b|v?5\.)`),
),
rule.Or(
rule.MatchingLanguages("Perl 6"),
rule.MatchingLanguages("Raku"),
regexp.MustCompile(`(?m)^\s*(?:use\s+v6\b|\bmodule\b|\b(?:my\s+)?class\b)`),
),
rule.Or(
@ -1105,6 +1173,10 @@ var ContentHeuristics = map[string]*Heuristics{
),
},
".pro": &Heuristics{
rule.Or(
rule.MatchingLanguages("Proguard"),
regexp.MustCompile(`(?m)^-(include\b.*\.pro$|keep\b|keepclassmembers\b|keepattributes\b)`),
),
rule.Or(
rule.MatchingLanguages("Prolog"),
regexp.MustCompile(`(?m)^[^\[#]+:-`),
@ -1216,6 +1288,12 @@ var ContentHeuristics = map[string]*Heuristics{
regexp.MustCompile(`(?m)#include|#pragma\s+(rs|version)|__attribute__`),
),
},
".s": &Heuristics{
rule.Or(
rule.MatchingLanguages("Motorola 68K Assembly"),
regexp.MustCompile(`(?m)(?im)\bmoveq(?:\.l)?\s+#(?:\$-?[0-9a-f]{1,3}|%[0-1]{1,8}|-?[0-9]{1,3}),\s*d[0-7]\b|(?im)^\s*move(?:\.[bwl])?\s+(?:sr|usp),\s*[^\s]+|(?im)^\s*move\.[bwl]\s+.*\b[ad]\d|(?im)^\s*movem\.[bwl]\b|(?im)^\s*move[mp](?:\.[wl])?\b|(?im)^\s*btst\b|(?im)^\s*dbra\b`),
),
},
".sc": &Heuristics{
rule.Or(
rule.MatchingLanguages("SuperCollider"),
@ -1267,7 +1345,7 @@ var ContentHeuristics = map[string]*Heuristics{
regexp.MustCompile(`(?m)\buse\s+(?:strict\b|v?5\.)`),
),
rule.Or(
rule.MatchingLanguages("Perl 6"),
rule.MatchingLanguages("Raku"),
regexp.MustCompile(`(?m)^\s*(?:use\s+v6\b|\bmodule\b|\b(?:my\s+)?class\b)`),
),
rule.Or(
@ -1313,13 +1391,27 @@ var ContentHeuristics = map[string]*Heuristics{
regexp.MustCompile(`(?m)(?i:^\s*<\?xml\s+version)`),
),
},
".v": &Heuristics{
rule.Or(
rule.MatchingLanguages("Coq"),
regexp.MustCompile(`(?m)\(\*.*?\*\)|(?:^|\s)(?:Proof|Qed)\.(?:$|\s)|(?:^|\s)Require[ \t]+Import\s`),
),
rule.Or(
rule.MatchingLanguages("Verilog"),
regexp.MustCompile(`(?m)^[ \t]*module\s+[^\s()]+\s+\#?\(|^[ \t]*`+"`"+`(?:ifdef|timescale)\s|^[ \t]*always[ \t]+@`),
),
rule.Or(
rule.MatchingLanguages("V"),
regexp.MustCompile(`(?m)\$(?:if|else)[ \t]|^[ \t]*fn\s+[^\s()]+\(.*?\).*?\{|^[ \t]*for\s*\{`),
),
},
".vba": &Heuristics{
rule.Or(
rule.MatchingLanguages("Vim script"),
regexp.MustCompile(`(?m)^UseVimball`),
),
rule.Always(
rule.MatchingLanguages("Visual Basic"),
rule.MatchingLanguages("VBA"),
),
},
".w": &Heuristics{
@ -1333,6 +1425,10 @@ var ContentHeuristics = map[string]*Heuristics{
),
},
".x": &Heuristics{
rule.Or(
rule.MatchingLanguages("DirectX 3D File"),
regexp.MustCompile(`(?m)^xof 030(2|3)(?:txt|bin|tzip|bzip)\b`),
),
rule.Or(
rule.MatchingLanguages("RPC"),
regexp.MustCompile(`(?m)\b(program|version)\s+\w+\s*{|\bunion\s+\w+\s+switch\s*\(`),

@ -1,5 +1,5 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data

@ -1,5 +1,5 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
@ -23,7 +23,10 @@ var LanguagesByFilename = map[string][]string{
".coffeelintignore": {"Ignore List"},
".cproject": {"XML"},
".cshrc": {"Shell"},
".curlrc": {"cURL Config"},
".cvsignore": {"Ignore List"},
".dir_colors": {"dircolors"},
".dircolors": {"dircolors"},
".dockerignore": {"Ignore List"},
".editorconfig": {"EditorConfig"},
".emacs": {"Emacs Lisp"},
@ -43,14 +46,17 @@ var LanguagesByFilename = map[string][]string{
".gvimrc": {"Vim script"},
".htaccess": {"ApacheConf"},
".htmlhintrc": {"JSON"},
".inputrc": {"Readline Config"},
".irbrc": {"Ruby"},
".jscsrc": {"JSON with Comments"},
".jshintrc": {"JSON with Comments"},
".jslintrc": {"JSON with Comments"},
".login": {"Shell"},
".luacheckrc": {"Lua"},
".nanorc": {"nanorc"},
".nodemonignore": {"Ignore List"},
".npmignore": {"Ignore List"},
".npmrc": {"NPM Config"},
".nvimrc": {"Vim script"},
".php": {"PHP"},
".php_cs": {"PHP"},
@ -67,6 +73,7 @@ var LanguagesByFilename = map[string][]string{
".viper": {"Emacs Lisp"},
".vscodeignore": {"Ignore List"},
".watchmanconfig": {"JSON"},
".wgetrc": {"Wget Config"},
".zlogin": {"Shell"},
".zlogout": {"Shell"},
".zprofile": {"Shell"},
@ -77,9 +84,9 @@ var LanguagesByFilename = map[string][]string{
"App.config": {"XML"},
"Appraisals": {"Ruby"},
"BSDmakefile": {"Makefile"},
"BUCK": {"Python"},
"BUILD": {"Python"},
"BUILD.bazel": {"Python"},
"BUCK": {"Starlark"},
"BUILD": {"Starlark"},
"BUILD.bazel": {"Starlark"},
"Berksfile": {"Ruby"},
"Brewfile": {"Ruby"},
"Buildfile": {"Ruby"},
@ -91,6 +98,8 @@ var LanguagesByFilename = map[string][]string{
"Capfile": {"Ruby"},
"Cargo.lock": {"TOML"},
"Cask": {"Emacs Lisp"},
"DEPS": {"Python"},
"DIR_COLORS": {"dircolors"},
"Dangerfile": {"Ruby"},
"Deliverfile": {"Ruby"},
"Dockerfile": {"Dockerfile"},
@ -144,11 +153,14 @@ var LanguagesByFilename = map[string][]string{
"Snapfile": {"Ruby"},
"Thorfile": {"Ruby"},
"Vagrantfile": {"Ruby"},
"WORKSPACE": {"Python"},
"WORKSPACE": {"Starlark"},
"Web.Debug.config": {"XML"},
"Web.Release.config": {"XML"},
"Web.config": {"XML"},
"XCompose": {"XCompose"},
"_curlrc": {"cURL Config"},
"_dir_colors": {"dircolors"},
"_dircolors": {"dircolors"},
"_emacs": {"Emacs Lisp"},
"_vimrc": {"Vim script"},
"abbrev_defs": {"Emacs Lisp"},
@ -173,7 +185,9 @@ var LanguagesByFilename = map[string][]string{
"delete.me": {"Text"},
"descrip.mmk": {"Module Management System"},
"descrip.mms": {"Module Management System"},
"dir_colors": {"dircolors"},
"encodings.dir": {"X Font Directory Index"},
"eqnrc": {"Roff"},
"expr-dist": {"R"},
"firestore.rules": {"Cloud Firestore Security Rules"},
"fonts.alias": {"X Font Directory Index"},
@ -189,6 +203,7 @@ var LanguagesByFilename = map[string][]string{
"gvimrc": {"Vim script"},
"haproxy.cfg": {"HAProxy"},
"httpd.conf": {"ApacheConf"},
"inputrc": {"Readline Config"},
"jsconfig.json": {"JSON with Comments"},
"keep.me": {"Text"},
"language-configuration.json": {"JSON with Comments"},
@ -212,6 +227,9 @@ var LanguagesByFilename = map[string][]string{
"nim.cfg": {"Nim"},
"nvimrc": {"Vim script"},
"owh": {"Tcl"},
"package.mask": {"Text"},
"package.use.mask": {"Text"},
"package.use.stable.mask": {"Text"},
"packages.config": {"XML"},
"pom.xml": {"Maven POM"},
"profile": {"Shell"},
@ -229,10 +247,15 @@ var LanguagesByFilename = map[string][]string{
"sshd_config": {"SSH Config"},
"starfield": {"Tcl"},
"test.me": {"Text"},
"troffrc": {"Roff"},
"troffrc-end": {"Roff"},
"tsconfig.json": {"JSON with Comments"},
"use.mask": {"Text"},
"use.stable.mask": {"Text"},
"vimrc": {"Vim script"},
"wscript": {"Python"},
"xcompose": {"XCompose"},
"yarn.lock": {"YAML"},
"zlogin": {"Shell"},
"zlogout": {"Shell"},
"zprofile": {"Shell"},

@ -0,0 +1,89 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
var LanguagesGroup = map[string]string{
"Alpine Abuild": "Shell",
"Apollo Guidance Computer": "Assembly",
"BibTeX": "TeX",
"Bison": "Yacc",
"Blade": "HTML",
"C2hs Haskell": "Haskell",
"Closure Templates": "HTML",
"ColdFusion CFC": "ColdFusion",
"Cython": "Python",
"ECLiPSe": "prolog",
"EJS": "HTML",
"Easybuild": "Python",
"Ecere Projects": "JavaScript",
"EditorConfig": "INI",
"Filterscript": "RenderScript",
"Gentoo Ebuild": "Shell",
"Gentoo Eclass": "Shell",
"Git Attributes": "INI",
"Git Config": "INI",
"Groovy Server Pages": "Groovy",
"HTML+Django": "HTML",
"HTML+ECR": "HTML",
"HTML+EEX": "HTML",
"HTML+ERB": "HTML",
"HTML+PHP": "HTML",
"HTML+Razor": "HTML",
"Haml": "HTML",
"Handlebars": "HTML",
"Ignore List": "INI",
"Isabelle ROOT": "Isabelle",
"JFlex": "Lex",
"JSON with Comments": "JSON",
"JSX": "JavaScript",
"Java Server Pages": "Java",
"JavaScript+ERB": "JavaScript",
"Jison": "Yacc",
"Jison Lex": "Lex",
"Latte": "HTML",
"Less": "CSS",
"Literate Agda": "Agda",
"Literate CoffeeScript": "CoffeeScript",
"Literate Haskell": "Haskell",
"M4Sugar": "M4",
"MUF": "Forth",
"Marko": "HTML",
"Motorola 68K Assembly": "Assembly",
"NPM Config": "INI",
"NumPy": "Python",
"OpenCL": "C",
"OpenRC runscript": "Shell",
"Parrot Assembly": "Parrot",
"Parrot Internal Representation": "Parrot",
"Pic": "Roff",
"PostCSS": "CSS",
"Pug": "HTML",
"Python console": "Python",
"Python traceback": "Python",
"RHTML": "HTML",
"Readline Config": "INI",
"Roff Manpage": "Roff",
"SCSS": "CSS",
"SSH Config": "INI",
"STON": "Smalltalk",
"Sage": "Python",
"Sass": "CSS",
"Scaml": "HTML",
"Slim": "HTML",
"Stylus": "CSS",
"SugarSS": "CSS",
"Svelte": "HTML",
"TSX": "TypeScript",
"Tcsh": "Shell",
"Twig": "HTML",
"Unified Parallel C": "C",
"Unix Assembly": "Assembly",
"Wget Config": "INI",
"X BitMap": "C",
"X PixMap": "C",
"XML Property List": "XML",
"cURL Config": "INI",
"fish": "Shell",
"nanorc": "INI",
}

@ -1,6 +1,6 @@
package data
import "github.com/src-d/enry/v2/data/rule"
import "github.com/go-enry/go-enry/v2/data/rule"
// Heuristics implements a rule-based content matching engine.

@ -1,5 +1,5 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
@ -20,6 +20,7 @@ var LanguagesByInterpreter = map[string][]string{
"coffee": {"CoffeeScript"},
"cperl": {"Perl"},
"crystal": {"Crystal"},
"csh": {"Tcsh"},
"csi": {"Scheme"},
"cvc4": {"SMT"},
"cwl-runner": {"Common Workflow Language"},
@ -36,6 +37,7 @@ var LanguagesByInterpreter = map[string][]string{
"gawk": {"Awk"},
"gerbv": {"Gerber Image"},
"gerbview": {"Gerber Image"},
"gjs": {"JavaScript"},
"gn": {"GN"},
"gnuplot": {"Gnuplot"},
"gosh": {"Scheme"},
@ -57,6 +59,7 @@ var LanguagesByInterpreter = map[string][]string{
"lua": {"Lua", "Terra"},
"macruby": {"Ruby"},
"make": {"Makefile"},
"makeinfo": {"Texinfo"},
"mathsat5": {"SMT"},
"mawk": {"Awk"},
"minised": {"sed"},
@ -77,7 +80,7 @@ var LanguagesByInterpreter = map[string][]string{
"parrot": {"Parrot Assembly", "Parrot Internal Representation"},
"pdksh": {"Shell"},
"perl": {"Perl", "Pod"},
"perl6": {"Perl 6", "Pod 6"},
"perl6": {"Pod 6", "Raku"},
"php": {"PHP"},
"picolisp": {"PicoLisp"},
"pike": {"Pike"},
@ -86,10 +89,13 @@ var LanguagesByInterpreter = map[string][]string{
"python": {"Python"},
"python2": {"Python"},
"python3": {"Python"},
"qjs": {"JavaScript"},
"qmake": {"QMake"},
"r6rs": {"Scheme"},
"racket": {"Racket"},
"rake": {"Ruby"},
"raku": {"Raku"},
"rakudo": {"Raku"},
"rbx": {"Ruby"},
"rc": {"Shell"},
"regina": {"REXX"},
@ -112,6 +118,7 @@ var LanguagesByInterpreter = map[string][]string{
"swipl": {"Prolog"},
"tcc": {"C"},
"tclsh": {"Tcl"},
"tcsh": {"Tcsh"},
"ts-node": {"TypeScript"},
"v8": {"JavaScript"},
"v8-shell": {"JavaScript"},

@ -1,5 +1,5 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
@ -13,6 +13,7 @@ var LanguagesMime = map[string]string{
"Ant Build System": "application/xml",
"Apex": "text/x-java",
"Asymptote": "text/x-kotlin",
"BibTeX": "text/x-stex",
"Brainfuck": "text/x-brainfuck",
"C": "text/x-csrc",
"C#": "text/x-csharp",
@ -50,7 +51,7 @@ var LanguagesMime = map[string]string{
"Easybuild": "text/x-python",
"Ecere Projects": "application/json",
"EditorConfig": "text/x-properties",
"Edje Data Collection": "application/json",
"Edje Data Collection": "text/x-c++src",
"Eiffel": "text/x-eiffel",
"Elm": "text/x-elm",
"Emacs Lisp": "text/x-common-lisp",
@ -146,7 +147,6 @@ var LanguagesMime = map[string]string{
"PLpgSQL": "text/x-sql",
"Pascal": "text/x-pascal",
"Perl": "text/x-perl",
"Perl 6": "text/x-perl",
"Pic": "text/troff",
"Pod": "text/x-perl",
"PowerShell": "application/x-powershell",
@ -161,6 +161,7 @@ var LanguagesMime = map[string]string{
"RHTML": "application/x-erb",
"RMarkdown": "text/x-gfm",
"RPM Spec": "text/x-rpm-spec",
"Raku": "text/x-perl",
"Reason": "text/x-rustsrc",
"Roff": "text/troff",
"Roff Manpage": "text/troff",
@ -174,6 +175,7 @@ var LanguagesMime = map[string]string{
"SQLPL": "text/x-sql",
"SRecode Template": "text/x-common-lisp",
"SVG": "text/xml",
"SWIG": "text/x-c++src",
"Sage": "text/x-python",
"SaltStack": "text/x-yaml",
"Sass": "text/x-sass",
@ -186,6 +188,7 @@ var LanguagesMime = map[string]string{
"Smarty": "text/x-smarty",
"Squirrel": "text/x-c++src",
"Standard ML": "text/x-ocaml",
"Starlark": "text/x-python",
"Svelte": "text/html",
"Swift": "text/x-swift",
"SystemVerilog": "text/x-systemverilog",
@ -203,9 +206,12 @@ var LanguagesMime = map[string]string{
"Unity3D Asset": "text/x-yaml",
"Uno": "text/x-csharp",
"UnrealScript": "text/x-java",
"V": "text/x-go",
"VBA": "text/x-vb",
"VBScript": "text/vbscript",
"VHDL": "text/x-vhdl",
"Verilog": "text/x-verilog",
"Visual Basic": "text/x-vb",
"Visual Basic .NET": "text/x-vb",
"Volt": "text/x-d",
"WebAssembly": "text/x-common-lisp",
"WebIDL": "text/x-webidl",
@ -214,6 +220,7 @@ var LanguagesMime = map[string]string{
"X PixMap": "text/x-csrc",
"XC": "text/x-csrc",
"XML": "text/xml",
"XML Property List": "text/xml",
"XPages": "text/xml",
"XProc": "text/xml",
"XQuery": "application/xquery",

@ -0,0 +1,560 @@
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
var LanguagesType = map[string]int{
"1C Enterprise": 2,
"4D": 2,
"ABAP": 2,
"ABNF": 1,
"AGS Script": 2,
"AMPL": 2,
"ANTLR": 2,
"API Blueprint": 3,
"APL": 2,
"ASN.1": 1,
"ASP": 2,
"ATS": 2,
"ActionScript": 2,
"Ada": 2,
"Adobe Font Metrics": 1,
"Agda": 2,
"Alloy": 2,
"Alpine Abuild": 2,
"Altium Designer": 1,
"AngelScript": 2,
"Ant Build System": 1,
"ApacheConf": 1,
"Apex": 2,
"Apollo Guidance Computer": 2,
"AppleScript": 2,
"Arc": 2,
"AsciiDoc": 4,
"AspectJ": 2,
"Assembly": 2,
"Asymptote": 2,
"Augeas": 2,
"AutoHotkey": 2,
"AutoIt": 2,
"Awk": 2,
"Ballerina": 2,
"Batchfile": 2,
"Befunge": 2,
"BibTeX": 3,
"Bison": 2,
"BitBake": 2,
"Blade": 3,
"BlitzBasic": 2,
"BlitzMax": 2,
"Bluespec": 2,
"Boo": 2,
"Brainfuck": 2,
"Brightscript": 2,
"C": 2,
"C#": 2,
"C++": 2,
"C-ObjDump": 1,
"C2hs Haskell": 2,
"CLIPS": 2,
"CMake": 2,
"COBOL": 2,
"COLLADA": 1,
"CSON": 1,
"CSS": 3,
"CSV": 1,
"CWeb": 2,
"Cabal Config": 1,
"Cap'n Proto": 2,
"CartoCSS": 2,
"Ceylon": 2,
"Chapel": 2,
"Charity": 2,
"ChucK": 2,
"Cirru": 2,
"Clarion": 2,
"Clean": 2,
"Click": 2,
"Clojure": 2,
"Closure Templates": 3,
"Cloud Firestore Security Rules": 1,
"CoNLL-U": 1,
"CodeQL": 2,
"CoffeeScript": 2,
"ColdFusion": 2,
"ColdFusion CFC": 2,
"Common Lisp": 2,
"Common Workflow Language": 2,
"Component Pascal": 2,
"Cool": 2,
"Coq": 2,
"Cpp-ObjDump": 1,
"Creole": 4,
"Crystal": 2,
"Csound": 2,
"Csound Document": 2,
"Csound Score": 2,
"Cuda": 2,
"Cycript": 2,
"Cython": 2,
"D": 2,
"D-ObjDump": 1,
"DIGITAL Command Language": 2,
"DM": 2,
"DNS Zone": 1,
"DTrace": 2,
"Darcs Patch": 1,
"Dart": 2,
"DataWeave": 2,
"Dhall": 2,
"Diff": 1,
"DirectX 3D File": 1,
"Dockerfile": 2,
"Dogescript": 2,
"Dylan": 2,
"E": 2,
"EBNF": 1,
"ECL": 2,
"ECLiPSe": 2,
"EJS": 3,
"EML": 1,
"EQ": 2,
"Eagle": 1,
"Easybuild": 1,
"Ecere Projects": 1,
"EditorConfig": 1,
"Edje Data Collection": 1,
"Eiffel": 2,
"Elixir": 2,
"Elm": 2,
"Emacs Lisp": 2,
"EmberScript": 2,
"Erlang": 2,
"F#": 2,
"F*": 2,
"FIGlet Font": 1,
"FLUX": 2,
"Factor": 2,
"Fancy": 2,
"Fantom": 2,
"Faust": 2,
"Filebench WML": 2,
"Filterscript": 2,
"Formatted": 1,
"Forth": 2,
"Fortran": 2,
"FreeMarker": 2,
"Frege": 2,
"G-code": 2,
"GAML": 2,
"GAMS": 2,
"GAP": 2,
"GCC Machine Description": 2,
"GDB": 2,
"GDScript": 2,
"GLSL": 2,
"GN": 1,
"Game Maker Language": 2,
"Genie": 2,
"Genshi": 2,
"Gentoo Ebuild": 2,
"Gentoo Eclass": 2,
"Gerber Image": 1,
"Gettext Catalog": 4,
"Gherkin": 2,
"Git Attributes": 1,
"Git Config": 1,
"Glyph": 2,
"Glyph Bitmap Distribution Format": 1,
"Gnuplot": 2,
"Go": 2,
"Golo": 2,
"Gosu": 2,
"Grace": 2,
"Gradle": 1,
"Grammatical Framework": 2,
"Graph Modeling Language": 1,
"GraphQL": 1,
"Graphviz (DOT)": 1,
"Groovy": 2,
"Groovy Server Pages": 2,
"HAProxy": 1,
"HCL": 2,
"HLSL": 2,
"HTML": 3,
"HTML+Django": 3,
"HTML+ECR": 3,
"HTML+EEX": 3,
"HTML+ERB": 3,
"HTML+PHP": 3,
"HTML+Razor": 3,
"HTTP": 1,
"HXML": 1,
"Hack": 2,
"Haml": 3,
"Handlebars": 3,
"Harbour": 2,
"Haskell": 2,
"Haxe": 2,
"HiveQL": 2,
"HolyC": 2,
"Hy": 2,
"HyPhy": 2,
"IDL": 2,
"IGOR Pro": 2,
"INI": 1,
"IRC log": 1,
"Idris": 2,
"Ignore List": 1,
"Inform 7": 2,
"Inno Setup": 2,
"Io": 2,
"Ioke": 2,
"Isabelle": 2,
"Isabelle ROOT": 2,
"J": 2,
"JFlex": 2,
"JSON": 1,
"JSON with Comments": 1,
"JSON5": 1,
"JSONLD": 1,
"JSONiq": 2,
"JSX": 2,
"Jasmin": 2,
"Java": 2,
"Java Properties": 1,
"Java Server Pages": 2,
"JavaScript": 2,
"JavaScript+ERB": 2,
"Jison": 2,
"Jison Lex": 2,
"Jolie": 2,
"Jsonnet": 2,
"Julia": 2,
"Jupyter Notebook": 3,
"KRL": 2,
"KiCad Layout": 1,
"KiCad Legacy Layout": 1,
"KiCad Schematic": 1,
"Kit": 3,
"Kotlin": 2,
"LFE": 2,
"LLVM": 2,
"LOLCODE": 2,
"LSL": 2,
"LTspice Symbol": 1,
"LabVIEW": 2,
"Lasso": 2,
"Latte": 3,
"Lean": 2,
"Less": 3,
"Lex": 2,
"LilyPond": 2,
"Limbo": 2,
"Linker Script": 1,
"Linux Kernel Module": 1,
"Liquid": 3,
"Literate Agda": 2,
"Literate CoffeeScript": 2,
"Literate Haskell": 2,
"LiveScript": 2,
"Logos": 2,
"Logtalk": 2,
"LookML": 2,
"LoomScript": 2,
"Lua": 2,
"M": 2,
"M4": 2,
"M4Sugar": 2,
"MATLAB": 2,
"MAXScript": 2,
"MLIR": 2,
"MQL4": 2,
"MQL5": 2,
"MTML": 3,
"MUF": 2,
"Makefile": 2,
"Mako": 2,
"Markdown": 4,
"Marko": 3,
"Mask": 3,
"Mathematica": 2,
"Maven POM": 1,
"Max": 2,
"MediaWiki": 4,
"Mercury": 2,
"Meson": 2,
"Metal": 2,
"Microsoft Developer Studio Project": 1,
"MiniD": 2,
"Mirah": 2,
"Modelica": 2,
"Modula-2": 2,
"Modula-3": 2,
"Module Management System": 2,
"Monkey": 2,
"Moocode": 2,
"MoonScript": 2,
"Motorola 68K Assembly": 2,
"Muse": 4,
"Myghty": 2,
"NASL": 2,
"NCL": 2,
"NL": 1,
"NPM Config": 1,
"NSIS": 2,
"Nearley": 2,
"Nemerle": 2,
"NetLinx": 2,
"NetLinx+ERB": 2,
"NetLogo": 2,
"NewLisp": 2,
"Nextflow": 2,
"Nginx": 1,
"Nim": 2,
"Ninja": 1,
"Nit": 2,
"Nix": 2,
"Nu": 2,
"NumPy": 2,
"OCaml": 2,
"ObjDump": 1,
"Object Data Instance Notation": 1,
"ObjectScript": 2,
"Objective-C": 2,
"Objective-C++": 2,
"Objective-J": 2,
"Odin": 2,
"Omgrofl": 2,
"Opa": 2,
"Opal": 2,
"Open Policy Agent": 2,
"OpenCL": 2,
"OpenEdge ABL": 2,
"OpenQASM": 2,
"OpenRC runscript": 2,
"OpenSCAD": 2,
"OpenStep Property List": 1,
"OpenType Feature File": 1,
"Org": 4,
"Ox": 2,
"Oxygene": 2,
"Oz": 2,
"P4": 2,
"PHP": 2,
"PLSQL": 2,
"PLpgSQL": 2,
"POV-Ray SDL": 2,
"Pan": 2,
"Papyrus": 2,
"Parrot": 2,
"Parrot Assembly": 2,
"Parrot Internal Representation": 2,
"Pascal": 2,
"Pawn": 2,
"Pep8": 2,
"Perl": 2,
"Pic": 3,
"Pickle": 1,
"PicoLisp": 2,
"PigLatin": 2,
"Pike": 2,
"PlantUML": 1,
"Pod": 4,
"Pod 6": 4,
"PogoScript": 2,
"Pony": 2,
"PostCSS": 3,
"PostScript": 3,
"PowerBuilder": 2,
"PowerShell": 2,
"Prisma": 1,
"Processing": 2,
"Proguard": 1,
"Prolog": 2,
"Propeller Spin": 2,
"Protocol Buffer": 1,
"Public Key": 1,
"Pug": 3,
"Puppet": 2,
"Pure Data": 1,
"PureBasic": 2,
"PureScript": 2,
"Python": 2,
"Python console": 2,
"Python traceback": 1,
"QML": 2,
"QMake": 2,
"Quake": 2,
"R": 2,
"RAML": 3,
"RDoc": 4,
"REALbasic": 2,
"REXX": 2,
"RHTML": 3,
"RMarkdown": 4,
"RPC": 2,
"RPM Spec": 1,
"RUNOFF": 3,
"Racket": 2,
"Ragel": 2,
"Raku": 2,
"Rascal": 2,
"Raw token data": 1,
"Readline Config": 1,
"Reason": 2,
"Rebol": 2,
"Red": 2,
"Redcode": 2,
"Regular Expression": 1,
"Ren'Py": 2,
"RenderScript": 2,
"Rich Text Format": 3,
"Ring": 2,
"Riot": 3,
"RobotFramework": 2,
"Roff": 3,
"Roff Manpage": 3,
"Rouge": 2,
"Ruby": 2,
"Rust": 2,
"SAS": 2,
"SCSS": 3,
"SMT": 2,
"SPARQL": 1,
"SQF": 2,
"SQL": 1,
"SQLPL": 2,
"SRecode Template": 3,
"SSH Config": 1,
"STON": 1,
"SVG": 1,
"SWIG": 2,
"Sage": 2,
"SaltStack": 2,
"Sass": 3,
"Scala": 2,
"Scaml": 3,
"Scheme": 2,
"Scilab": 2,
"Self": 2,
"ShaderLab": 2,
"Shell": 2,
"ShellSession": 2,
"Shen": 2,
"Slash": 2,
"Slice": 2,
"Slim": 3,
"SmPL": 2,
"Smali": 2,
"Smalltalk": 2,
"Smarty": 2,
"Solidity": 2,
"SourcePawn": 2,
"Spline Font Database": 1,
"Squirrel": 2,
"Stan": 2,
"Standard ML": 2,
"Starlark": 2,
"Stata": 2,
"Stylus": 3,
"SubRip Text": 1,
"SugarSS": 3,
"SuperCollider": 2,
"Svelte": 3,
"Swift": 2,
"SystemVerilog": 2,
"TI Program": 2,
"TLA": 2,
"TOML": 1,
"TSQL": 2,
"TSX": 2,
"TXL": 2,
"Tcl": 2,
"Tcsh": 2,
"TeX": 3,
"Tea": 3,
"Terra": 2,
"Texinfo": 4,
"Text": 4,
"Textile": 4,
"Thrift": 2,
"Turing": 2,
"Turtle": 1,
"Twig": 3,
"Type Language": 1,
"TypeScript": 2,
"Unified Parallel C": 2,
"Unity3D Asset": 1,
"Unix Assembly": 2,
"Uno": 2,
"UnrealScript": 2,
"UrWeb": 2,
"V": 2,
"VBA": 2,
"VBScript": 2,
"VCL": 2,
"VHDL": 2,
"Vala": 2,
"Verilog": 2,
"Vim Snippet": 3,
"Vim script": 2,
"Visual Basic .NET": 2,
"Volt": 2,
"Vue": 3,
"Wavefront Material": 1,
"Wavefront Object": 1,
"Web Ontology Language": 1,
"WebAssembly": 2,
"WebIDL": 2,
"WebVTT": 1,
"Wget Config": 1,
"Windows Registry Entries": 1,
"Wollok": 2,
"World of Warcraft Addon Data": 1,
"X BitMap": 1,
"X Font Directory Index": 1,
"X PixMap": 1,
"X10": 2,
"XC": 2,
"XCompose": 1,
"XML": 1,
"XML Property List": 1,
"XPages": 1,
"XProc": 2,
"XQuery": 2,
"XS": 2,
"XSLT": 2,
"Xojo": 2,
"Xtend": 2,
"YAML": 1,
"YANG": 1,
"YARA": 2,
"YASnippet": 3,
"Yacc": 2,
"ZAP": 2,
"ZIL": 2,
"Zeek": 2,
"ZenScript": 2,
"Zephir": 2,
"Zig": 2,
"Zimpl": 2,
"cURL Config": 1,
"desktop": 1,
"dircolors": 1,
"eC": 2,
"edn": 1,
"fish": 2,
"mIRC Script": 2,
"mcfunction": 2,
"mupad": 2,
"nanorc": 1,
"nesC": 2,
"ooc": 2,
"q": 2,
"reStructuredText": 4,
"sed": 2,
"wdl": 2,
"wisp": 2,
"xBase": 2,
}

@ -1,5 +1,5 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
// Code generated by github.com/go-enry/go-enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 40992ba7f86889f80dfed3ba95e11e1082200bad
package data
@ -19,8 +19,11 @@ var VendorMatchers = substring.Or(
substring.Regexp(`(^|/)ltsugar.m4`),
substring.Regexp(`(^|/)ltversion.m4`),
substring.Regexp(`(^|/)lt~obsolete.m4`),
substring.Regexp(`dotnet-install\.(ps1|sh)$`),
substring.Regexp(`cpplint.py`),
substring.Regexp(`node_modules/`),
substring.Regexp(`(^|/)\.yarn/releases/`),
substring.Regexp(`(^|/)_esy$`),
substring.Regexp(`bower_components/`),
substring.Regexp(`^rebar$`),
substring.Regexp(`erlang.mk`),

@ -11,6 +11,6 @@
Upstream Linguist YAML files are used to generate datastructures for data
package.
*/
package enry // import "github.com/src-d/enry/v2"
package enry // import "github.com/go-enry/go-enry/v2"
//go:generate make code-generate

@ -1,11 +1,11 @@
module github.com/src-d/enry/v2
module github.com/go-enry/go-enry/v2
go 1.12
go 1.14
require (
github.com/src-d/go-oniguruma v1.1.0
github.com/go-enry/go-oniguruma v1.2.0
github.com/stretchr/testify v1.3.0
github.com/toqueteos/trie v1.0.0 // indirect
gopkg.in/toqueteos/substring.v1 v1.0.2
gopkg.in/yaml.v2 v2.2.2
gopkg.in/yaml.v2 v2.2.8
)

@ -1,9 +1,10 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-enry/go-oniguruma v1.2.0 h1:oBO9XC1IDT9+AoWW5oFsa/7gFeOPacEqDbyXZKWXuDs=
github.com/go-enry/go-oniguruma v1.2.0/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/src-d/go-oniguruma v1.1.0 h1:EG+Nm5n2JqWUaCjtM0NtutPxU7ZN5Tp50GWrrV8bTww=
github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@ -13,5 +14,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/toqueteos/substring.v1 v1.0.2 h1:urLqCeMm6x/eTuQa1oZerNw8N1KNOIp5hD5kGL7lFsE=
gopkg.in/toqueteos/substring.v1 v1.0.2/go.mod h1:Eb2Z1UYehlVK8LYW2WBVR2rwbujsz3aX8XDrM1vbNew=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

@ -5,7 +5,7 @@ package tokenizer
import (
"bytes"
"github.com/src-d/enry/v2/regex"
"github.com/go-enry/go-enry/v2/regex"
)
// Tokenize returns lexical tokens from content. The tokens returned match what
@ -18,10 +18,6 @@ func Tokenize(content []byte) []string {
content = content[:ByteLimit]
}
// Copy the input so that changes wrought by the tokenization steps do not
// modify the caller's copy of the input. See #196.
content = append([]byte(nil), content...)
tokens := make([][]byte, 0, 50)
for _, extract := range extractTokens {
var extractedTokens [][]byte
@ -162,7 +158,7 @@ func extractAndReplaceSGML(content []byte) ([]byte, [][]byte) {
continue
}
token := append(match[1], '>')
token := append(append([]byte(nil), match[1]...), '>')
SGMLTokens = append(SGMLTokens, token)
attributes := getSGMLAttributes(match[0])
SGMLTokens = append(SGMLTokens, attributes...)

@ -2,7 +2,7 @@
package tokenizer
import "github.com/src-d/enry/v2/internal/tokenizer/flex"
import "github.com/go-enry/go-enry/v2/internal/tokenizer/flex"
// Tokenize returns lexical tokens from content. The tokens returned match what
// the Linguist library returns. At most the first ByteLimit bytes of content are tokenized.

@ -3,7 +3,7 @@
package regex
import (
rubex "github.com/src-d/go-oniguruma"
rubex "github.com/go-enry/go-oniguruma"
)
type EnryRegexp = *rubex.Regexp

@ -5,7 +5,7 @@ import (
"path/filepath"
"strings"
"github.com/src-d/enry/v2/data"
"github.com/go-enry/go-enry/v2/data"
)
const binSniffLen = 8000
@ -80,5 +80,9 @@ func GetColor(language string) string {
return color
}
if color, ok := data.LanguagesColor[GetLanguageGroup(language)]; ok {
return color
}
return "#cccccc"
}

@ -1,5 +1,4 @@
## go-oniguruma
<a href="https://travis-ci.org/src-d/go-oniguruma"><img alt="Build Status" src="https://travis-ci.org/src-d/go-oniguruma.svg?branch=master" /></a>
# go-oniguruma ![Test](https://github.com/go-enry/go-oniguruma/workflows/Test/badge.svg)
This repository is a fork of [moovweb/rubex](https://github.com/moovweb/rubex/tree/go1) - a simple regular expression library (based on [oniguruma](https://github.com/kkos/oniguruma)) that supports Ruby's regex syntax.
@ -8,7 +7,9 @@ The _rubex_ was originally created by Zhigang Chen (zhigang.chen@moovweb.com or
By the benchmark tests in regexp, the library is 40% to 10X faster than Regexp on all but one test. Unlike Go's regexp, this library supports named capture groups and also allow `"\\1"` and `"\\k<name>"` in replacement strings.
The library calls the _oniguruma_ regex library for regex pattern searching. All replacement code is done in Go.
### Install all (_oniguruma_ and _rubex_):
Install
-------
```sh
# linux (debian/ubuntu/...)
sudo apt-get install libonig-dev
@ -16,5 +17,10 @@ sudo apt-get install libonig-dev
# osx (homebrew)
brew install oniguruma
go install -i .
go get github.com/go-enry/go-oniguruma
```
License
-------
Apache License Version 2.0, see [LICENSE](LICENSE)

@ -0,0 +1,3 @@
module github.com/go-enry/go-oniguruma
go 1.14

@ -1,61 +0,0 @@
# source{d} Contributing Guidelines
source{d} projects accept contributions via GitHub pull requests.
This document outlines some of the
conventions on development workflow, commit message formatting, contact points,
and other resources to make it easier to get your contribution accepted.
## Certificate of Origin
By contributing to this project, you agree to the [Developer Certificate of
Origin (DCO)](DCO). This document was created by the Linux Kernel community and is a
simple statement that you, as a contributor, have the legal right to make the
contribution.
In order to show your agreement with the DCO you should include at the end of the commit message,
the following line: `Signed-off-by: John Doe <john.doe@example.com>`, using your real name.
This can be done easily using the [`-s`](https://github.com/git/git/blob/b2c150d3aa82f6583b9aadfecc5f8fa1c74aca09/Documentation/git-commit.txt#L154-L161) flag on the `git commit`.
If you find yourself pushed a few commits without `Signed-off-by`, you can still add it afterwards. We wrote a manual which can help: [fix-DCO.md](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md).
## Support Channels
The official support channels, for both users and contributors, are:
- GitHub issues: each repository has its own list of issues.
- Slack: join the [source{d} Slack](https://join.slack.com/t/sourced-community/shared_invite/enQtMjc4Njk5MzEyNzM2LTFjNzY4NjEwZGEwMzRiNTM4MzRlMzQ4MmIzZjkwZmZlM2NjODUxZmJjNDI1OTcxNDAyMmZlNmFjODZlNTg0YWM) community.
*Before opening a new issue or submitting a new pull request, it's helpful to
search the project - it's likely that another user has already reported the
issue you're facing, or it's a known issue that we're already aware of.
## How to Contribute
Pull Requests (PRs) are the main and exclusive way to contribute code to source{d} projects.
In order for a PR to be accepted it needs to pass this list of requirements:
- The contribution must be correctly explained with natural language and providing a minimum working example that reproduces it.
- All PRs must be written idiomaticly:
- for Go: formatted according to [gofmt](https://golang.org/cmd/gofmt/), and without any warnings from [go lint](https://github.com/golang/lint) nor [go vet](https://golang.org/cmd/vet/)
- for other languages, similar constraints apply.
- They should in general include tests, and those shall pass.
- If the PR is a bug fix, it has to include a new unit test that fails before the patch is merged.
- If the PR is a new feature, it has to come with a suite of unit tests, that tests the new functionality.
- In any case, all the PRs have to pass the personal evaluation of at least one of the [maintainers](MAINTAINERS) of the project.
### Format of the commit message
Every commit message should describe what was changed, under which context and, if applicable, the GitHub issue it relates to:
```
plumbing: packp, Skip argument validations for unknown capabilities. Fixes #623
```
The format can be described more formally as follows:
```
<package>: <subpackage>, <what changed>. [Fixes #<issue-number>]
```

@ -1,25 +0,0 @@
Developer's Certificate of Origin 1.1
By making a contribution to this project, I certify that:
(a) The contribution was created in whole or in part by me and I
have the right to submit it under the open source license
indicated in the file; or
(b) The contribution is based upon previous work that, to the best
of my knowledge, is covered under an appropriate open source
license and I have the right under that license to submit that
work with modifications, whether created in whole or in part
by me, under the same open source license (unless I am
permitted to submit under a different license), as indicated
in the file; or
(c) The contribution was provided directly to me by some other
person who certified (a), (b) or (c) and I have not modified
it.
(d) I understand and agree that this project and the contribution
are public and that a record of the contribution (including all
personal information I submit with it, including my sign-off) is
maintained indefinitely and may be redistributed consistent with
this project or the open source license(s) involved.

@ -1 +0,0 @@
Alexander Bezzubov <alex@sourced.tech> (@bzz)

@ -1,328 +0,0 @@
# enry [![GoDoc](https://godoc.org/github.com/src-d/enry?status.svg)](https://godoc.org/github.com/src-d/enry) [![Build Status](https://travis-ci.com/src-d/enry.svg?branch=master)](https://travis-ci.com/src-d/enry) [![codecov](https://codecov.io/gh/src-d/enry/branch/master/graph/badge.svg)](https://codecov.io/gh/src-d/enry)
File programming language detector and toolbox to ignore binary or vendored files. *enry*, started as a port to _Go_ of the original [linguist](https://github.com/github/linguist) _Ruby_ library, that has an improved *2x performance*.
* [Installation](#installation)
* [Examples](#examples)
* [CLI](#cli)
* [Java bindings](#java-bindings)
* [Python bindings](#python-bindings)
* [Divergences from linguist](#divergences-from-linguist)
* [Benchmarks](#benchmarks)
* [Why Enry?](#why-enry)
* [Development](#development)
* [Sync with github/linguist upstream](#sync-with-githublinguist-upstream)
* [Misc](#misc)
* [Benchmark](#benchmark)
* [Faster regexp engine (optional)](#faster-regexp-engine-optional)
* [License](#license)
Installation
------------
The recommended way to install enry is to either [download a release](https://github.com/src-d/enry/releases) or
```
go get github.com/src-d/enry/cmd/enry
```
This project is now part of [source{d} Engine](https://sourced.tech/engine),
which provides the simplest way to get started with a single command.
Visit [sourced.tech/engine](https://sourced.tech/engine) for more information.
Examples
------------
```go
lang, safe := enry.GetLanguageByExtension("foo.go")
fmt.Println(lang, safe)
// result: Go true
lang, safe := enry.GetLanguageByContent("foo.m", []byte("<matlab-code>"))
fmt.Println(lang, safe)
// result: Matlab true
lang, safe := enry.GetLanguageByContent("bar.m", []byte("<objective-c-code>"))
fmt.Println(lang, safe)
// result: Objective-C true
// all strategies together
lang := enry.GetLanguage("foo.cpp", []byte("<cpp-code>"))
// result: C++ true
```
Note that the returned boolean value `safe` is set either to `true`, if there is only one possible language detected, or to `false` otherwise.
To get a list of possible languages for a given file, you can use the plural version of the detecting functions.
```go
langs := enry.GetLanguages("foo.h", []byte("<cpp-code>"))
// result: []string{"C", "C++", "Objective-C}
langs := enry.GetLanguagesByExtension("foo.asc", []byte("<content>"), nil)
// result: []string{"AGS Script", "AsciiDoc", "Public Key"}
langs := enry.GetLanguagesByFilename("Gemfile", []byte("<content>"), []string{})
// result: []string{"Ruby"}
```
CLI
------------
You can use enry as a command,
```bash
$ enry --help
enry v2.0.0 build: 05-08-2019_20_40_35 commit: 6ccf0b6, based on linguist commit: e456098
enry, A simple (and faster) implementation of github/linguist
usage: enry [-mode=(file|line|byte)] [-prog] <path>
enry [-mode=(file|line|byte)] [-prog] [-json] [-breakdown] <path>
enry [-mode=(file|line|byte)] [-prog] [-json] [-breakdown]
enry [-version]
```
and on repository root, it'll return an output similar to *linguist*'s output,
```bash
$ enry
97.71% Go
1.60% C
0.31% Shell
0.22% Java
0.07% Ruby
0.05% Makefile
0.04% Scala
0.01% Gnuplot
```
but not only the output; its flags are also the same as *linguist*'s ones,
```bash
$ enry --breakdown
97.71% Go
1.60% C
0.31% Shell
0.22% Java
0.07% Ruby
0.05% Makefile
0.04% Scala
0.01% Gnuplot
Scala
java/build.sbt
java/project/plugins.sbt
Java
java/src/main/java/tech/sourced/enry/Enry.java
java/src/main/java/tech/sourced/enry/GoUtils.java
java/src/main/java/tech/sourced/enry/Guess.java
java/src/test/java/tech/sourced/enry/EnryTest.java
Makefile
Makefile
java/Makefile
Go
benchmark_test.go
```
even the JSON flag,
```bash
$ enry --json | jq .
{
"C": [
"internal/tokenizer/flex/lex.linguist_yy.c",
"internal/tokenizer/flex/lex.linguist_yy.h",
"internal/tokenizer/flex/linguist.h",
"python/_c_enry.c",
"python/enry.c"
],
"Gnuplot": [
"benchmarks/plot-histogram.gp"
],
"Go": [
"benchmark_test.go",
```
Note that enry's CLI **_doesn't need a git repository to work_**, which is intentionally different from the linguist.
## Java bindings
Generated Java bindings using a C shared library and JNI are available under [`java`](https://github.com/src-d/enry/blob/master/java) and published on Maven at [tech.sourced:enry-java](https://mvnrepository.com/artifact/tech.sourced/enry-java) for macOS and linux.
## Python bindings
Generated Python bindings using a C shared library and cffi are not available yet and are WIP under [src-d/enry#154](https://github.com/src-d/enry/issues/154).
Divergences from linguist
------------
The `enry` library is based on the data from `github/linguist` version **v7.5.1**.
As opposed to linguist, `enry` [CLI tool](#cli) does *not* require a full Git repository in the filesystem in order to report languages.
Parsing [linguist/samples](https://github.com/github/linguist/tree/master/samples) the following `enry` results are different from linguist:
* [Heuristics for ".es" extension](https://github.com/github/linguist/blob/e761f9b013e5b61161481fcb898b59721ee40e3d/lib/linguist/heuristics.yml#L103) in JavaScript could not be parsed, due to unsupported backreference in RE2 regexp engine.
* [Heuristics for ".rno" extension](https://github.com/github/linguist/blob/3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d/lib/linguist/heuristics.yml#L365) in RUNOFF could not be parsed, due to unsupported lookahead in RE2 regexp engine.
* As of [Linguist v5.3.2](https://github.com/github/linguist/releases/tag/v5.3.2) it is using [flex-based scanner in C for tokenization](https://github.com/github/linguist/pull/3846). Enry still uses [extract_token](https://github.com/github/linguist/pull/3846/files#diff-d5179df0b71620e3fac4535cd1368d15L60) regex-based algorithm. See [#193](https://github.com/src-d/enry/issues/193).
* Bayesian classifier can't distinguish "SQL" from "PLpgSQL. See [#194](https://github.com/src-d/enry/issues/194).
* Detection of [generated files](https://github.com/github/linguist/blob/bf95666fc15e49d556f2def4d0a85338423c25f3/lib/linguist/generated.rb#L53) is not supported yet.
(Thus they are not excluded from CLI output). See [#213](https://github.com/src-d/enry/issues/213).
* XML detection strategy is not implemented. See [#192](https://github.com/src-d/enry/issues/192).
* Overriding languages and types though `.gitattributes` is not yet supported. See [#18](https://github.com/src-d/enry/issues/18).
* `enry` CLI output does NOT exclude `.gitignore`ed files and git submodules, as linguist does
In all the cases above that have an issue number - we plan to update enry to match Linguist behavior.
Benchmarks
------------
Enry's language detection has been compared with Linguist's one. In order to do that, Linguist's project directory [*linguist/samples*](https://github.com/github/linguist/tree/master/samples) was used as a set of files to run benchmarks against.
We got these results:
![histogram](benchmarks/histogram/distribution.png)
The histogram shows the number of files detected (y-axis) per time interval bucket (x-axis). As one can see, most of the files were detected faster by enry.
We found few cases where enry turns slower than linguist due to
Go regexp engine being slower than Ruby's, based on [oniguruma](https://github.com/kkos/oniguruma) library, written in C.
See [instructions](#misc) for running enry with oniguruma.
Why Enry?
------------
In the movie [My Fair Lady](https://en.wikipedia.org/wiki/My_Fair_Lady), [Professor Henry Higgins](http://www.imdb.com/character/ch0011719/?ref_=tt_cl_t2) is one of the main characters. Henry is a linguist and at the very beginning of the movie enjoys guessing the origin of people based on their accent.
"Enry Iggins" is how [Eliza Doolittle](http://www.imdb.com/character/ch0011720/?ref_=tt_cl_t1), [pronounces](https://www.youtube.com/watch?v=pwNKyTktDIE) the name of the Professor during the first half of the movie.
## Development
To build enry's CLI run:
make build
this will generate a binary in the project's root directory called `enry`.
To run the tests:
make test
### Sync with github/linguist upstream
*enry* re-uses parts of the original [github/linguist](https://github.com/github/linguist) to generate internal data structures.
In order to update to the latest release of linguist do:
```bash
$ git clone https://github.com/github/linguist.git .linguist
$ cd .linguist; git checkout <release-tag>; cd ..
# put the new release's commit sha in the generator_test.go (to re-generate .gold test fixtures)
# https://github.com/src-d/enry/blob/13d3d66d37a87f23a013246a1b0678c9ee3d524b/internal/code-generator/generator/generator_test.go#L18
$ make code-generate
```
To stay in sync, enry needs to be updated when a new release of the linguist includes changes to any of the following files:
* [languages.yml](https://github.com/github/linguist/blob/master/lib/linguist/languages.yml)
* [heuristics.yml](https://github.com/github/linguist/blob/master/lib/linguist/heuristics.yml)
* [vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml)
* [documentation.yml](https://github.com/github/linguist/blob/master/lib/linguist/documentation.yml)
There is no automation for detecting the changes in the linguist project, so this process above has to be done manually from time to time.
When submitting a pull request syncing up to a new release, please make sure it only contains the changes in
the generated files (in [data](https://github.com/src-d/enry/blob/master/data) subdirectory).
Separating all the necessary "manual" code changes to a different PR that includes some background description and an update to the documentation on ["divergences from linguist"](##divergences-from-linguist) is very much appreciated as it simplifies the maintenance (review/release notes/etc).
## Misc
<details>
### Benchmark
All benchmark scripts are in [*benchmarks*](https://github.com/src-d/enry/blob/master/benchmarks) directory.
#### Dependencies
As benchmarks depend on Ruby and Github-Linguist gem make sure you have:
- Ruby (e.g using [`rbenv`](https://github.com/rbenv/rbenv)), [`bundler`](https://bundler.io/) installed
- Docker
- [native dependencies](https://github.com/github/linguist/#dependencies) installed
- Build the gem `cd .linguist && bundle install && rake build_gem && cd -`
- Install it `gem install --no-rdoc --no-ri --local .linguist/github-linguist-*.gem`
#### Quick benchmark
To run quicker benchmarks you can either:
make benchmarks
to get average times for the main detection function and strategies for the whole samples set or:
make benchmarks-samples
if you want to see measures per sample file.
#### Full benchmark
If you want to reproduce the same benchmarks as reported above:
- Make sure all [dependencies](#benchmark-dependencies) are installed
- Install [gnuplot](http://gnuplot.info) (in order to plot the histogram)
- Run `ENRY_TEST_REPO="$PWD/.linguist" benchmarks/run.sh` (takes ~15h)
It will run the benchmarks for enry and linguist, parse the output, create csv files and plot the histogram.
### Faster regexp engine (optional)
[Oniguruma](https://github.com/kkos/oniguruma) is CRuby's regular expression engine.
It is very fast and performs better than the one built into Go runtime. *enry* supports swapping
between those two engines thanks to [rubex](https://github.com/moovweb/rubex) project.
The typical overall speedup from using Oniguruma is 1.5-2x. However, it requires CGo and the external shared library.
On macOS with [Homebrew](https://brew.sh/), it is:
```
brew install oniguruma
```
On Ubuntu, it is
```
sudo apt install libonig-dev
```
To build enry with Oniguruma regexps use the `oniguruma` build tag
```
go get -v -t --tags oniguruma ./...
```
and then rebuild the project.
</details>
License
------------
Apache License, Version 2.0. See [LICENSE](LICENSE)

@ -1,783 +0,0 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
package data
import "strings"
// LanguageByAliasMap keeps alias for different languages and use the name of the languages as an alias too.
// All the keys (alias or not) are written in lower case and the whitespaces has been replaced by underscores.
var LanguageByAliasMap = map[string]string{
"1c_enterprise": "1C Enterprise",
"abap": "ABAP",
"abl": "OpenEdge ABL",
"abnf": "ABNF",
"abuild": "Alpine Abuild",
"acfm": "Adobe Font Metrics",
"aconf": "ApacheConf",
"actionscript": "ActionScript",
"actionscript3": "ActionScript",
"actionscript_3": "ActionScript",
"ada": "Ada",
"ada2005": "Ada",
"ada95": "Ada",
"adobe_composite_font_metrics": "Adobe Font Metrics",
"adobe_font_metrics": "Adobe Font Metrics",
"adobe_multiple_font_metrics": "Adobe Font Metrics",
"advpl": "xBase",
"afdko": "OpenType Feature File",
"agda": "Agda",
"ags": "AGS Script",
"ags_script": "AGS Script",
"ahk": "AutoHotkey",
"alloy": "Alloy",
"alpine_abuild": "Alpine Abuild",
"altium": "Altium Designer",
"altium_designer": "Altium Designer",
"amfm": "Adobe Font Metrics",
"ampl": "AMPL",
"angelscript": "AngelScript",
"ant_build_system": "Ant Build System",
"antlr": "ANTLR",
"apache": "ApacheConf",
"apacheconf": "ApacheConf",
"apex": "Apex",
"api_blueprint": "API Blueprint",
"apkbuild": "Alpine Abuild",
"apl": "APL",
"apollo_guidance_computer": "Apollo Guidance Computer",
"applescript": "AppleScript",
"arc": "Arc",
"arexx": "REXX",
"as3": "ActionScript",
"asciidoc": "AsciiDoc",
"asm": "Assembly",
"asn.1": "ASN.1",
"asp": "ASP",
"aspectj": "AspectJ",
"aspx": "ASP",
"aspx-vb": "ASP",
"assembly": "Assembly",
"asymptote": "Asymptote",
"ats": "ATS",
"ats2": "ATS",
"au3": "AutoIt",
"augeas": "Augeas",
"autoconf": "M4Sugar",
"autohotkey": "AutoHotkey",
"autoit": "AutoIt",
"autoit3": "AutoIt",
"autoitscript": "AutoIt",
"awk": "Awk",
"b3d": "BlitzBasic",
"ballerina": "Ballerina",
"bash": "Shell",
"bash_session": "ShellSession",
"bat": "Batchfile",
"batch": "Batchfile",
"batchfile": "Batchfile",
"befunge": "Befunge",
"bison": "Bison",
"bitbake": "BitBake",
"blade": "Blade",
"blitz3d": "BlitzBasic",
"blitzbasic": "BlitzBasic",
"blitzmax": "BlitzMax",
"blitzplus": "BlitzBasic",
"bluespec": "Bluespec",
"bmax": "BlitzMax",
"boo": "Boo",
"bplus": "BlitzBasic",
"brainfuck": "Brainfuck",
"brightscript": "Brightscript",
"bro": "Zeek",
"bsdmake": "Makefile",
"byond": "DM",
"c": "C",
"c#": "C#",
"c++": "C++",
"c++-objdump": "Cpp-ObjDump",
"c-objdump": "C-ObjDump",
"c2hs": "C2hs Haskell",
"c2hs_haskell": "C2hs Haskell",
"cabal": "Cabal Config",
"cabal_config": "Cabal Config",
"cap'n_proto": "Cap'n Proto",
"carto": "CartoCSS",
"cartocss": "CartoCSS",
"ceylon": "Ceylon",
"cfc": "ColdFusion CFC",
"cfm": "ColdFusion",
"cfml": "ColdFusion",
"chapel": "Chapel",
"charity": "Charity",
"chpl": "Chapel",
"chuck": "ChucK",
"cirru": "Cirru",
"clarion": "Clarion",
"clean": "Clean",
"click": "Click",
"clipper": "xBase",
"clips": "CLIPS",
"clojure": "Clojure",
"closure_templates": "Closure Templates",
"cloud_firestore_security_rules": "Cloud Firestore Security Rules",
"cmake": "CMake",
"cobol": "COBOL",
"coffee": "CoffeeScript",
"coffee-script": "CoffeeScript",
"coffeescript": "CoffeeScript",
"coldfusion": "ColdFusion",
"coldfusion_cfc": "ColdFusion CFC",
"coldfusion_html": "ColdFusion",
"collada": "COLLADA",
"common_lisp": "Common Lisp",
"common_workflow_language": "Common Workflow Language",
"component_pascal": "Component Pascal",
"conll": "CoNLL-U",
"conll-u": "CoNLL-U",
"conll-x": "CoNLL-U",
"console": "ShellSession",
"cool": "Cool",
"coq": "Coq",
"cperl": "Perl",
"cpp": "C++",
"cpp-objdump": "Cpp-ObjDump",
"creole": "Creole",
"crystal": "Crystal",
"csharp": "C#",
"cson": "CSON",
"csound": "Csound",
"csound-csd": "Csound Document",
"csound-orc": "Csound",
"csound-sco": "Csound Score",
"csound_document": "Csound Document",
"csound_score": "Csound Score",
"css": "CSS",
"csv": "CSV",
"cucumber": "Gherkin",
"cuda": "Cuda",
"cweb": "CWeb",
"cwl": "Common Workflow Language",
"cycript": "Cycript",
"cython": "Cython",
"d": "D",
"d-objdump": "D-ObjDump",
"darcs_patch": "Darcs Patch",
"dart": "Dart",
"dataweave": "DataWeave",
"dcl": "DIGITAL Command Language",
"delphi": "Component Pascal",
"desktop": "desktop",
"dhall": "Dhall",
"diff": "Diff",
"digital_command_language": "DIGITAL Command Language",
"django": "HTML+Django",
"dm": "DM",
"dns_zone": "DNS Zone",
"dockerfile": "Dockerfile",
"dogescript": "Dogescript",
"dosbatch": "Batchfile",
"dosini": "INI",
"dpatch": "Darcs Patch",
"dtrace": "DTrace",
"dtrace-script": "DTrace",
"dylan": "Dylan",
"e": "E",
"eagle": "Eagle",
"easybuild": "Easybuild",
"ebnf": "EBNF",
"ec": "eC",
"ecere_projects": "Ecere Projects",
"ecl": "ECL",
"eclipse": "ECLiPSe",
"ecr": "HTML+ECR",
"editor-config": "EditorConfig",
"editorconfig": "EditorConfig",
"edje_data_collection": "Edje Data Collection",
"edn": "edn",
"eeschema_schematic": "KiCad Schematic",
"eex": "HTML+EEX",
"eiffel": "Eiffel",
"ejs": "EJS",
"elisp": "Emacs Lisp",
"elixir": "Elixir",
"elm": "Elm",
"emacs": "Emacs Lisp",
"emacs_lisp": "Emacs Lisp",
"emberscript": "EmberScript",
"eml": "EML",
"eq": "EQ",
"erb": "HTML+ERB",
"erlang": "Erlang",
"f#": "F#",
"f*": "F*",
"factor": "Factor",
"fancy": "Fancy",
"fantom": "Fantom",
"figfont": "FIGlet Font",
"figlet_font": "FIGlet Font",
"filebench_wml": "Filebench WML",
"filterscript": "Filterscript",
"fish": "fish",
"flex": "Lex",
"flux": "FLUX",
"formatted": "Formatted",
"forth": "Forth",
"fortran": "Fortran",
"foxpro": "xBase",
"freemarker": "FreeMarker",
"frege": "Frege",
"fsharp": "F#",
"fstar": "F*",
"ftl": "FreeMarker",
"fundamental": "Text",
"g-code": "G-code",
"game_maker_language": "Game Maker Language",
"gams": "GAMS",
"gap": "GAP",
"gcc_machine_description": "GCC Machine Description",
"gdb": "GDB",
"gdscript": "GDScript",
"genie": "Genie",
"genshi": "Genshi",
"gentoo_ebuild": "Gentoo Ebuild",
"gentoo_eclass": "Gentoo Eclass",
"gerber_image": "Gerber Image",
"gettext_catalog": "Gettext Catalog",
"gf": "Grammatical Framework",
"gherkin": "Gherkin",
"git-ignore": "Ignore List",
"git_attributes": "Git Attributes",
"git_config": "Git Config",
"gitattributes": "Git Attributes",
"gitconfig": "Git Config",
"gitignore": "Ignore List",
"gitmodules": "Git Config",
"glsl": "GLSL",
"glyph": "Glyph",
"glyph_bitmap_distribution_format": "Glyph Bitmap Distribution Format",
"gn": "GN",
"gnuplot": "Gnuplot",
"go": "Go",
"golang": "Go",
"golo": "Golo",
"gosu": "Gosu",
"grace": "Grace",
"gradle": "Gradle",
"grammatical_framework": "Grammatical Framework",
"graph_modeling_language": "Graph Modeling Language",
"graphql": "GraphQL",
"graphviz_(dot)": "Graphviz (DOT)",
"groff": "Roff",
"groovy": "Groovy",
"groovy_server_pages": "Groovy Server Pages",
"gsp": "Groovy Server Pages",
"hack": "Hack",
"haml": "Haml",
"handlebars": "Handlebars",
"haproxy": "HAProxy",
"harbour": "Harbour",
"haskell": "Haskell",
"haxe": "Haxe",
"hbs": "Handlebars",
"hcl": "HCL",
"hiveql": "HiveQL",
"hlsl": "HLSL",
"holyc": "HolyC",
"html": "HTML",
"html+django": "HTML+Django",
"html+django/jinja": "HTML+Django",
"html+ecr": "HTML+ECR",
"html+eex": "HTML+EEX",
"html+erb": "HTML+ERB",
"html+jinja": "HTML+Django",
"html+php": "HTML+PHP",
"html+razor": "HTML+Razor",
"html+ruby": "RHTML",
"htmlbars": "Handlebars",
"htmldjango": "HTML+Django",
"http": "HTTP",
"hxml": "HXML",
"hy": "Hy",
"hylang": "Hy",
"hyphy": "HyPhy",
"i7": "Inform 7",
"idl": "IDL",
"idris": "Idris",
"ignore": "Ignore List",
"ignore_list": "Ignore List",
"igor": "IGOR Pro",
"igor_pro": "IGOR Pro",
"igorpro": "IGOR Pro",
"inc": "PHP",
"inform7": "Inform 7",
"inform_7": "Inform 7",
"ini": "INI",
"inno_setup": "Inno Setup",
"io": "Io",
"ioke": "Ioke",
"ipython_notebook": "Jupyter Notebook",
"irc": "IRC log",
"irc_log": "IRC log",
"irc_logs": "IRC log",
"isabelle": "Isabelle",
"isabelle_root": "Isabelle ROOT",
"j": "J",
"jasmin": "Jasmin",
"java": "Java",
"java_properties": "Java Properties",
"java_server_page": "Groovy Server Pages",
"java_server_pages": "Java Server Pages",
"javascript": "JavaScript",
"javascript+erb": "JavaScript+ERB",
"jflex": "JFlex",
"jison": "Jison",
"jison_lex": "Jison Lex",
"jolie": "Jolie",
"jruby": "Ruby",
"js": "JavaScript",
"json": "JSON",
"json5": "JSON5",
"json_with_comments": "JSON with Comments",
"jsonc": "JSON with Comments",
"jsoniq": "JSONiq",
"jsonld": "JSONLD",
"jsonnet": "Jsonnet",
"jsp": "Java Server Pages",
"jsx": "JSX",
"julia": "Julia",
"jupyter_notebook": "Jupyter Notebook",
"kicad_layout": "KiCad Layout",
"kicad_legacy_layout": "KiCad Legacy Layout",
"kicad_schematic": "KiCad Schematic",
"kit": "Kit",
"kotlin": "Kotlin",
"krl": "KRL",
"labview": "LabVIEW",
"lasso": "Lasso",
"lassoscript": "Lasso",
"latex": "TeX",
"latte": "Latte",
"lean": "Lean",
"less": "Less",
"lex": "Lex",
"lfe": "LFE",
"lhaskell": "Literate Haskell",
"lhs": "Literate Haskell",
"lilypond": "LilyPond",
"limbo": "Limbo",
"linker_script": "Linker Script",
"linux_kernel_module": "Linux Kernel Module",
"liquid": "Liquid",
"lisp": "Common Lisp",
"litcoffee": "Literate CoffeeScript",
"literate_agda": "Literate Agda",
"literate_coffeescript": "Literate CoffeeScript",
"literate_haskell": "Literate Haskell",
"live-script": "LiveScript",
"livescript": "LiveScript",
"llvm": "LLVM",
"logos": "Logos",
"logtalk": "Logtalk",
"lolcode": "LOLCODE",
"lookml": "LookML",
"loomscript": "LoomScript",
"ls": "LiveScript",
"lsl": "LSL",
"ltspice_symbol": "LTspice Symbol",
"lua": "Lua",
"m": "M",
"m4": "M4",
"m4sugar": "M4Sugar",
"macruby": "Ruby",
"make": "Makefile",
"makefile": "Makefile",
"mako": "Mako",
"man": "Roff",
"man-page": "Roff",
"man_page": "Roff",
"manpage": "Roff",
"markdown": "Markdown",
"marko": "Marko",
"markojs": "Marko",
"mask": "Mask",
"mathematica": "Mathematica",
"matlab": "MATLAB",
"maven_pom": "Maven POM",
"max": "Max",
"max/msp": "Max",
"maxmsp": "Max",
"maxscript": "MAXScript",
"mcfunction": "mcfunction",
"mdoc": "Roff",
"mediawiki": "MediaWiki",
"mercury": "Mercury",
"meson": "Meson",
"metal": "Metal",
"mf": "Makefile",
"minid": "MiniD",
"mirah": "Mirah",
"mma": "Mathematica",
"modelica": "Modelica",
"modula-2": "Modula-2",
"modula-3": "Modula-3",
"module_management_system": "Module Management System",
"monkey": "Monkey",
"moocode": "Moocode",
"moonscript": "MoonScript",
"motorola_68k_assembly": "Motorola 68K Assembly",
"mql4": "MQL4",
"mql5": "MQL5",
"mtml": "MTML",
"muf": "MUF",
"mumps": "M",
"mupad": "mupad",
"myghty": "Myghty",
"nanorc": "nanorc",
"nasm": "Assembly",
"ncl": "NCL",
"nearley": "Nearley",
"nemerle": "Nemerle",
"nesc": "nesC",
"netlinx": "NetLinx",
"netlinx+erb": "NetLinx+ERB",
"netlogo": "NetLogo",
"newlisp": "NewLisp",
"nextflow": "Nextflow",
"nginx": "Nginx",
"nginx_configuration_file": "Nginx",
"nim": "Nim",
"ninja": "Ninja",
"nit": "Nit",
"nix": "Nix",
"nixos": "Nix",
"njk": "HTML+Django",
"nl": "NL",
"node": "JavaScript",
"nroff": "Roff",
"nsis": "NSIS",
"nu": "Nu",
"numpy": "NumPy",
"nunjucks": "HTML+Django",
"nush": "Nu",
"nvim": "Vim script",
"obj-c": "Objective-C",
"obj-c++": "Objective-C++",
"obj-j": "Objective-J",
"objc": "Objective-C",
"objc++": "Objective-C++",
"objdump": "ObjDump",
"objective-c": "Objective-C",
"objective-c++": "Objective-C++",
"objective-j": "Objective-J",
"objectivec": "Objective-C",
"objectivec++": "Objective-C++",
"objectivej": "Objective-J",
"objectpascal": "Component Pascal",
"objectscript": "ObjectScript",
"objj": "Objective-J",
"ocaml": "OCaml",
"octave": "MATLAB",
"omgrofl": "Omgrofl",
"oncrpc": "RPC",
"ooc": "ooc",
"opa": "Opa",
"opal": "Opal",
"opencl": "OpenCL",
"openedge": "OpenEdge ABL",
"openedge_abl": "OpenEdge ABL",
"openrc": "OpenRC runscript",
"openrc_runscript": "OpenRC runscript",
"openscad": "OpenSCAD",
"opentype_feature_file": "OpenType Feature File",
"org": "Org",
"osascript": "AppleScript",
"ox": "Ox",
"oxygene": "Oxygene",
"oz": "Oz",
"p4": "P4",
"pan": "Pan",
"pandoc": "Markdown",
"papyrus": "Papyrus",
"parrot": "Parrot",
"parrot_assembly": "Parrot Assembly",
"parrot_internal_representation": "Parrot Internal Representation",
"pascal": "Pascal",
"pasm": "Parrot Assembly",
"pawn": "Pawn",
"pcbnew": "KiCad Layout",
"pep8": "Pep8",
"perl": "Perl",
"perl6": "Perl 6",
"perl_6": "Perl 6",
"php": "PHP",
"pic": "Pic",
"pickle": "Pickle",
"picolisp": "PicoLisp",
"piglatin": "PigLatin",
"pike": "Pike",
"pir": "Parrot Internal Representation",
"plpgsql": "PLpgSQL",
"plsql": "PLSQL",
"pod": "Pod",
"pod_6": "Pod 6",
"pogoscript": "PogoScript",
"pony": "Pony",
"posh": "PowerShell",
"postcss": "PostCSS",
"postscr": "PostScript",
"postscript": "PostScript",
"pot": "Gettext Catalog",
"pov-ray": "POV-Ray SDL",
"pov-ray_sdl": "POV-Ray SDL",
"povray": "POV-Ray SDL",
"powerbuilder": "PowerBuilder",
"powershell": "PowerShell",
"processing": "Processing",
"progress": "OpenEdge ABL",
"prolog": "Prolog",
"propeller_spin": "Propeller Spin",
"protobuf": "Protocol Buffer",
"protocol_buffer": "Protocol Buffer",
"protocol_buffers": "Protocol Buffer",
"public_key": "Public Key",
"pug": "Pug",
"puppet": "Puppet",
"pure_data": "Pure Data",
"purebasic": "PureBasic",
"purescript": "PureScript",
"pwsh": "PowerShell",
"pycon": "Python console",
"pyrex": "Cython",
"python": "Python",
"python3": "Python",
"python_console": "Python console",
"python_traceback": "Python traceback",
"q": "q",
"qmake": "QMake",
"qml": "QML",
"quake": "Quake",
"r": "R",
"racket": "Racket",
"ragel": "Ragel",
"ragel-rb": "Ragel",
"ragel-ruby": "Ragel",
"rake": "Ruby",
"raml": "RAML",
"rascal": "Rascal",
"raw": "Raw token data",
"raw_token_data": "Raw token data",
"razor": "HTML+Razor",
"rb": "Ruby",
"rbx": "Ruby",
"rdoc": "RDoc",
"realbasic": "REALbasic",
"reason": "Reason",
"rebol": "Rebol",
"red": "Red",
"red/system": "Red",
"redcode": "Redcode",
"regex": "Regular Expression",
"regexp": "Regular Expression",
"regular_expression": "Regular Expression",
"ren'py": "Ren'Py",
"renderscript": "RenderScript",
"renpy": "Ren'Py",
"restructuredtext": "reStructuredText",
"rexx": "REXX",
"rhtml": "RHTML",
"rich_text_format": "Rich Text Format",
"ring": "Ring",
"rmarkdown": "RMarkdown",
"robotframework": "RobotFramework",
"roff": "Roff",
"roff_manpage": "Roff Manpage",
"rouge": "Rouge",
"rpc": "RPC",
"rpcgen": "RPC",
"rpm_spec": "RPM Spec",
"rs-274x": "Gerber Image",
"rscript": "R",
"rss": "XML",
"rst": "reStructuredText",
"ruby": "Ruby",
"runoff": "RUNOFF",
"rust": "Rust",
"rusthon": "Python",
"sage": "Sage",
"salt": "SaltStack",
"saltstack": "SaltStack",
"saltstate": "SaltStack",
"sas": "SAS",
"sass": "Sass",
"scala": "Scala",
"scaml": "Scaml",
"scheme": "Scheme",
"scilab": "Scilab",
"scss": "SCSS",
"sed": "sed",
"self": "Self",
"sh": "Shell",
"shaderlab": "ShaderLab",
"shell": "Shell",
"shell-script": "Shell",
"shellsession": "ShellSession",
"shen": "Shen",
"slash": "Slash",
"slice": "Slice",
"slim": "Slim",
"smali": "Smali",
"smalltalk": "Smalltalk",
"smarty": "Smarty",
"sml": "Standard ML",
"smt": "SMT",
"snippet": "YASnippet",
"solidity": "Solidity",
"sourcemod": "SourcePawn",
"sourcepawn": "SourcePawn",
"soy": "Closure Templates",
"sparql": "SPARQL",
"specfile": "RPM Spec",
"spline_font_database": "Spline Font Database",
"splus": "R",
"sqf": "SQF",
"sql": "SQL",
"sqlpl": "SQLPL",
"squeak": "Smalltalk",
"squirrel": "Squirrel",
"srecode_template": "SRecode Template",
"ssh_config": "SSH Config",
"stan": "Stan",
"standard_ml": "Standard ML",
"stata": "Stata",
"ston": "STON",
"stylus": "Stylus",
"subrip_text": "SubRip Text",
"sugarss": "SugarSS",
"supercollider": "SuperCollider",
"svelte": "Svelte",
"svg": "SVG",
"swift": "Swift",
"systemverilog": "SystemVerilog",
"tcl": "Tcl",
"tcsh": "Tcsh",
"tea": "Tea",
"terra": "Terra",
"terraform": "HCL",
"tex": "TeX",
"text": "Text",
"textile": "Textile",
"thrift": "Thrift",
"ti_program": "TI Program",
"tl": "Type Language",
"tla": "TLA",
"toml": "TOML",
"troff": "Roff",
"ts": "TypeScript",
"tsql": "TSQL",
"tsx": "TSX",
"turing": "Turing",
"turtle": "Turtle",
"twig": "Twig",
"txl": "TXL",
"type_language": "Type Language",
"typescript": "TypeScript",
"udiff": "Diff",
"unified_parallel_c": "Unified Parallel C",
"unity3d_asset": "Unity3D Asset",
"unix_assembly": "Unix Assembly",
"uno": "Uno",
"unrealscript": "UnrealScript",
"ur": "UrWeb",
"ur/web": "UrWeb",
"urweb": "UrWeb",
"vala": "Vala",
"vb.net": "Visual Basic",
"vbnet": "Visual Basic",
"vcl": "VCL",
"verilog": "Verilog",
"vhdl": "VHDL",
"vim": "Vim script",
"vim_script": "Vim script",
"viml": "Vim script",
"visual_basic": "Visual Basic",
"volt": "Volt",
"vue": "Vue",
"wasm": "WebAssembly",
"wast": "WebAssembly",
"wavefront_material": "Wavefront Material",
"wavefront_object": "Wavefront Object",
"wdl": "wdl",
"web_ontology_language": "Web Ontology Language",
"webassembly": "WebAssembly",
"webidl": "WebIDL",
"webvtt": "WebVTT",
"winbatch": "Batchfile",
"windows_registry_entries": "Windows Registry Entries",
"wisp": "wisp",
"wollok": "Wollok",
"world_of_warcraft_addon_data": "World of Warcraft Addon Data",
"wsdl": "XML",
"x10": "X10",
"x_bitmap": "X BitMap",
"x_font_directory_index": "X Font Directory Index",
"x_pixmap": "X PixMap",
"xbase": "xBase",
"xbm": "X BitMap",
"xc": "XC",
"xcompose": "XCompose",
"xdr": "RPC",
"xhtml": "HTML",
"xml": "XML",
"xml+genshi": "Genshi",
"xml+kid": "Genshi",
"xojo": "Xojo",
"xpages": "XPages",
"xpm": "X PixMap",
"xproc": "XProc",
"xquery": "XQuery",
"xs": "XS",
"xsd": "XML",
"xsl": "XSLT",
"xslt": "XSLT",
"xten": "X10",
"xtend": "Xtend",
"yacc": "Yacc",
"yaml": "YAML",
"yang": "YANG",
"yara": "YARA",
"yas": "YASnippet",
"yasnippet": "YASnippet",
"yml": "YAML",
"zap": "ZAP",
"zeek": "Zeek",
"zenscript": "ZenScript",
"zephir": "Zephir",
"zig": "Zig",
"zil": "ZIL",
"zimpl": "Zimpl",
"zsh": "Shell",
}
// LanguageByAlias looks up the language name by it's alias or name.
// It mirrors the logic of github linguist and is needed e.g for heuristcs.yml
// that mixes names and aliases in a language field (see XPM example).
func LanguageByAlias(langOrAlias string) (lang string, ok bool) {
k := convertToAliasKey(langOrAlias)
lang, ok = LanguageByAliasMap[k]
return
}
// convertToAliasKey converts language name to a key in LanguageByAliasMap.
// Following
// - internal.code-generator.generator.convertToAliasKey()
// - GetLanguageByAlias()
// conventions.
// It is here to avoid dependency on "generate" and "enry" packages.
func convertToAliasKey(langName string) string {
ak := strings.SplitN(langName, `,`, 2)[0]
ak = strings.Replace(ak, ` `, `_`, -1)
ak = strings.ToLower(ak)
return ak
}

@ -1,7 +0,0 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
package data
// linguist's commit from which files were generated.
var LinguistCommit = "3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d"

@ -1,526 +0,0 @@
// Code generated by github.com/src-d/enry/v2/internal/code-generator DO NOT EDIT.
// Extracted from github/linguist commit: 3a1bd3c3d3e741a8aaec4704f782e06f5cd2a00d
package data
var LanguagesType = map[string]int{
"1C Enterprise": 2,
"ABAP": 2,
"ABNF": 1,
"AGS Script": 2,
"AMPL": 2,
"ANTLR": 2,
"API Blueprint": 3,
"APL": 2,
"ASN.1": 1,
"ASP": 2,
"ATS": 2,
"ActionScript": 2,
"Ada": 2,
"Adobe Font Metrics": 1,
"Agda": 2,
"Alloy": 2,
"Alpine Abuild": 2,
"Altium Designer": 1,
"AngelScript": 2,
"Ant Build System": 1,
"ApacheConf": 1,
"Apex": 2,
"Apollo Guidance Computer": 2,
"AppleScript": 2,
"Arc": 2,
"AsciiDoc": 4,
"AspectJ": 2,
"Assembly": 2,
"Asymptote": 2,
"Augeas": 2,
"AutoHotkey": 2,
"AutoIt": 2,
"Awk": 2,
"Ballerina": 2,
"Batchfile": 2,
"Befunge": 2,
"Bison": 2,
"BitBake": 2,
"Blade": 3,
"BlitzBasic": 2,
"BlitzMax": 2,
"Bluespec": 2,
"Boo": 2,
"Brainfuck": 2,
"Brightscript": 2,
"C": 2,
"C#": 2,
"C++": 2,
"C-ObjDump": 1,
"C2hs Haskell": 2,
"CLIPS": 2,
"CMake": 2,
"COBOL": 2,
"COLLADA": 1,
"CSON": 1,
"CSS": 3,
"CSV": 1,
"CWeb": 2,
"Cabal Config": 1,
"Cap'n Proto": 2,
"CartoCSS": 2,
"Ceylon": 2,
"Chapel": 2,
"Charity": 2,
"ChucK": 2,
"Cirru": 2,
"Clarion": 2,
"Clean": 2,
"Click": 2,
"Clojure": 2,
"Closure Templates": 3,
"Cloud Firestore Security Rules": 1,
"CoNLL-U": 1,
"CoffeeScript": 2,
"ColdFusion": 2,
"ColdFusion CFC": 2,
"Common Lisp": 2,
"Common Workflow Language": 2,
"Component Pascal": 2,
"Cool": 2,
"Coq": 2,
"Cpp-ObjDump": 1,
"Creole": 4,
"Crystal": 2,
"Csound": 2,
"Csound Document": 2,
"Csound Score": 2,
"Cuda": 2,
"Cycript": 2,
"Cython": 2,
"D": 2,
"D-ObjDump": 1,
"DIGITAL Command Language": 2,
"DM": 2,
"DNS Zone": 1,
"DTrace": 2,
"Darcs Patch": 1,
"Dart": 2,
"DataWeave": 2,
"Dhall": 2,
"Diff": 1,
"Dockerfile": 2,
"Dogescript": 2,
"Dylan": 2,
"E": 2,
"EBNF": 1,
"ECL": 2,
"ECLiPSe": 2,
"EJS": 3,
"EML": 1,
"EQ": 2,
"Eagle": 1,
"Easybuild": 1,
"Ecere Projects": 1,
"EditorConfig": 1,
"Edje Data Collection": 1,
"Eiffel": 2,
"Elixir": 2,
"Elm": 2,
"Emacs Lisp": 2,
"EmberScript": 2,
"Erlang": 2,
"F#": 2,
"F*": 2,
"FIGlet Font": 1,
"FLUX": 2,
"Factor": 2,
"Fancy": 2,
"Fantom": 2,
"Filebench WML": 2,
"Filterscript": 2,
"Formatted": 1,
"Forth": 2,
"Fortran": 2,
"FreeMarker": 2,
"Frege": 2,
"G-code": 2,
"GAMS": 2,
"GAP": 2,
"GCC Machine Description": 2,
"GDB": 2,
"GDScript": 2,
"GLSL": 2,
"GN": 1,
"Game Maker Language": 2,
"Genie": 2,
"Genshi": 2,
"Gentoo Ebuild": 2,
"Gentoo Eclass": 2,
"Gerber Image": 1,
"Gettext Catalog": 4,
"Gherkin": 2,
"Git Attributes": 1,
"Git Config": 1,
"Glyph": 2,
"Glyph Bitmap Distribution Format": 1,
"Gnuplot": 2,
"Go": 2,
"Golo": 2,
"Gosu": 2,
"Grace": 2,
"Gradle": 1,
"Grammatical Framework": 2,
"Graph Modeling Language": 1,
"GraphQL": 1,
"Graphviz (DOT)": 1,
"Groovy": 2,
"Groovy Server Pages": 2,
"HAProxy": 1,
"HCL": 2,
"HLSL": 2,
"HTML": 3,
"HTML+Django": 3,
"HTML+ECR": 3,
"HTML+EEX": 3,
"HTML+ERB": 3,
"HTML+PHP": 3,
"HTML+Razor": 3,
"HTTP": 1,
"HXML": 1,
"Hack": 2,
"Haml": 3,
"Handlebars": 3,
"Harbour": 2,
"Haskell": 2,
"Haxe": 2,
"HiveQL": 2,
"HolyC": 2,
"Hy": 2,
"HyPhy": 2,
"IDL": 2,
"IGOR Pro": 2,
"INI": 1,
"IRC log": 1,
"Idris": 2,
"Ignore List": 1,
"Inform 7": 2,
"Inno Setup": 2,
"Io": 2,
"Ioke": 2,
"Isabelle": 2,
"Isabelle ROOT": 2,
"J": 2,
"JFlex": 2,
"JSON": 1,
"JSON with Comments": 1,
"JSON5": 1,
"JSONLD": 1,
"JSONiq": 2,
"JSX": 2,
"Jasmin": 2,
"Java": 2,
"Java Properties": 1,
"Java Server Pages": 2,
"JavaScript": 2,
"JavaScript+ERB": 2,
"Jison": 2,
"Jison Lex": 2,
"Jolie": 2,
"Jsonnet": 2,
"Julia": 2,
"Jupyter Notebook": 3,
"KRL": 2,
"KiCad Layout": 1,
"KiCad Legacy Layout": 1,
"KiCad Schematic": 1,
"Kit": 3,
"Kotlin": 2,
"LFE": 2,
"LLVM": 2,
"LOLCODE": 2,
"LSL": 2,
"LTspice Symbol": 1,
"LabVIEW": 2,
"Lasso": 2,
"Latte": 3,
"Lean": 2,
"Less": 3,
"Lex": 2,
"LilyPond": 2,
"Limbo": 2,
"Linker Script": 1,
"Linux Kernel Module": 1,
"Liquid": 3,
"Literate Agda": 2,
"Literate CoffeeScript": 2,
"Literate Haskell": 2,
"LiveScript": 2,
"Logos": 2,
"Logtalk": 2,
"LookML": 2,
"LoomScript": 2,
"Lua": 2,
"M": 2,
"M4": 2,
"M4Sugar": 2,
"MATLAB": 2,
"MAXScript": 2,
"MQL4": 2,
"MQL5": 2,
"MTML": 3,
"MUF": 2,
"Makefile": 2,
"Mako": 2,
"Markdown": 4,
"Marko": 3,
"Mask": 3,
"Mathematica": 2,
"Maven POM": 1,
"Max": 2,
"MediaWiki": 4,
"Mercury": 2,
"Meson": 2,
"Metal": 2,
"MiniD": 2,
"Mirah": 2,
"Modelica": 2,
"Modula-2": 2,
"Modula-3": 2,
"Module Management System": 2,
"Monkey": 2,
"Moocode": 2,
"MoonScript": 2,
"Motorola 68K Assembly": 2,
"Myghty": 2,
"NCL": 2,
"NL": 1,
"NSIS": 2,
"Nearley": 2,
"Nemerle": 2,
"NetLinx": 2,
"NetLinx+ERB": 2,
"NetLogo": 2,
"NewLisp": 2,
"Nextflow": 2,
"Nginx": 1,
"Nim": 2,
"Ninja": 1,
"Nit": 2,
"Nix": 2,
"Nu": 2,
"NumPy": 2,
"OCaml": 2,
"ObjDump": 1,
"ObjectScript": 2,
"Objective-C": 2,
"Objective-C++": 2,
"Objective-J": 2,
"Omgrofl": 2,
"Opa": 2,
"Opal": 2,
"OpenCL": 2,
"OpenEdge ABL": 2,
"OpenRC runscript": 2,
"OpenSCAD": 2,
"OpenType Feature File": 1,
"Org": 4,
"Ox": 2,
"Oxygene": 2,
"Oz": 2,
"P4": 2,
"PHP": 2,
"PLSQL": 2,
"PLpgSQL": 2,
"POV-Ray SDL": 2,
"Pan": 2,
"Papyrus": 2,
"Parrot": 2,
"Parrot Assembly": 2,
"Parrot Internal Representation": 2,
"Pascal": 2,
"Pawn": 2,
"Pep8": 2,
"Perl": 2,
"Perl 6": 2,
"Pic": 3,
"Pickle": 1,
"PicoLisp": 2,
"PigLatin": 2,
"Pike": 2,
"Pod": 4,
"Pod 6": 4,
"PogoScript": 2,
"Pony": 2,
"PostCSS": 3,
"PostScript": 3,
"PowerBuilder": 2,
"PowerShell": 2,
"Processing": 2,
"Prolog": 2,
"Propeller Spin": 2,
"Protocol Buffer": 1,
"Public Key": 1,
"Pug": 3,
"Puppet": 2,
"Pure Data": 1,
"PureBasic": 2,
"PureScript": 2,
"Python": 2,
"Python console": 2,
"Python traceback": 1,
"QML": 2,
"QMake": 2,
"Quake": 2,
"R": 2,
"RAML": 3,
"RDoc": 4,
"REALbasic": 2,
"REXX": 2,
"RHTML": 3,
"RMarkdown": 4,
"RPC": 2,
"RPM Spec": 1,
"RUNOFF": 3,
"Racket": 2,
"Ragel": 2,
"Rascal": 2,
"Raw token data": 1,
"Reason": 2,
"Rebol": 2,
"Red": 2,
"Redcode": 2,
"Regular Expression": 1,
"Ren'Py": 2,
"RenderScript": 2,
"Rich Text Format": 3,
"Ring": 2,
"RobotFramework": 2,
"Roff": 3,
"Roff Manpage": 3,
"Rouge": 2,
"Ruby": 2,
"Rust": 2,
"SAS": 2,
"SCSS": 3,
"SMT": 2,
"SPARQL": 1,
"SQF": 2,
"SQL": 1,
"SQLPL": 2,
"SRecode Template": 3,
"SSH Config": 1,
"STON": 1,
"SVG": 1,
"Sage": 2,
"SaltStack": 2,
"Sass": 3,
"Scala": 2,
"Scaml": 3,
"Scheme": 2,
"Scilab": 2,
"Self": 2,
"ShaderLab": 2,
"Shell": 2,
"ShellSession": 2,
"Shen": 2,
"Slash": 2,
"Slice": 2,
"Slim": 3,
"Smali": 2,
"Smalltalk": 2,
"Smarty": 2,
"Solidity": 2,
"SourcePawn": 2,
"Spline Font Database": 1,
"Squirrel": 2,
"Stan": 2,
"Standard ML": 2,
"Stata": 2,
"Stylus": 3,
"SubRip Text": 1,
"SugarSS": 3,
"SuperCollider": 2,
"Svelte": 3,
"Swift": 2,
"SystemVerilog": 2,
"TI Program": 2,
"TLA": 2,
"TOML": 1,
"TSQL": 2,
"TSX": 2,
"TXL": 2,
"Tcl": 2,
"Tcsh": 2,
"TeX": 3,
"Tea": 3,
"Terra": 2,
"Text": 4,
"Textile": 4,
"Thrift": 2,
"Turing": 2,
"Turtle": 1,
"Twig": 3,
"Type Language": 1,
"TypeScript": 2,
"Unified Parallel C": 2,
"Unity3D Asset": 1,
"Unix Assembly": 2,
"Uno": 2,
"UnrealScript": 2,
"UrWeb": 2,
"VCL": 2,
"VHDL": 2,
"Vala": 2,
"Verilog": 2,
"Vim script": 2,
"Visual Basic": 2,
"Volt": 2,
"Vue": 3,
"Wavefront Material": 1,
"Wavefront Object": 1,
"Web Ontology Language": 1,
"WebAssembly": 2,
"WebIDL": 2,
"WebVTT": 1,
"Windows Registry Entries": 1,
"Wollok": 2,
"World of Warcraft Addon Data": 1,
"X BitMap": 1,
"X Font Directory Index": 1,
"X PixMap": 1,
"X10": 2,
"XC": 2,
"XCompose": 1,
"XML": 1,
"XPages": 1,
"XProc": 2,
"XQuery": 2,
"XS": 2,
"XSLT": 2,
"Xojo": 2,
"Xtend": 2,
"YAML": 1,
"YANG": 1,
"YARA": 2,
"YASnippet": 3,
"Yacc": 2,
"ZAP": 2,
"ZIL": 2,
"Zeek": 2,
"ZenScript": 2,
"Zephir": 2,
"Zig": 2,
"Zimpl": 2,
"desktop": 1,
"eC": 2,
"edn": 1,
"fish": 2,
"mcfunction": 2,
"mupad": 2,
"nanorc": 1,
"nesC": 2,
"ooc": 2,
"q": 2,
"reStructuredText": 4,
"sed": 2,
"wdl": 2,
"wisp": 2,
"xBase": 2,
}

@ -1,20 +0,0 @@
dist: trusty
language: go
go:
- '1.11.x'
- '1.12.x'
env:
global:
- LD_LIBRARY_PATH="/usr/local/lib":${LD_LIBRARY_PATH}
- GO111MODULE=on
- ONIGURUMA_VERSION='6.9.1'
before_install: # install oniguruma manually as trusty has only ancient 5.x
- sudo apt-get install -y dpkg # dpkg >= 1.17.5ubuntu5.8 fixes https://bugs.launchpad.net/ubuntu/+source/dpkg/+bug/1730627
- wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig5_${ONIGURUMA_VERSION}-1_amd64.deb"
- sudo dpkg -i "libonig5_${ONIGURUMA_VERSION}-1_amd64.deb"
- wget "http://archive.ubuntu.com/ubuntu/pool/universe/libo/libonig/libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
- sudo dpkg -i "libonig-dev_${ONIGURUMA_VERSION}-1_amd64.deb"
script:
- go test -v --cover -race

@ -1 +0,0 @@
module github.com/src-d/go-oniguruma

@ -1,12 +1,16 @@
language: go
go:
- 1.4
- 1.5
- 1.6
- 1.7
- 1.8
- 1.9
- tip
- "1.4.x"
- "1.5.x"
- "1.6.x"
- "1.7.x"
- "1.8.x"
- "1.9.x"
- "1.10.x"
- "1.11.x"
- "1.12.x"
- "1.13.x"
- "tip"
go_import_path: gopkg.in/yaml.v2

14
vendor/gopkg.in/yaml.v2/decode.go generated vendored

@ -319,10 +319,14 @@ func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unm
}
const (
// 400,000 decode operations is ~500kb of dense object declarations, or ~5kb of dense object declarations with 10000% alias expansion
// 400,000 decode operations is ~500kb of dense object declarations, or
// ~5kb of dense object declarations with 10000% alias expansion
alias_ratio_range_low = 400000
// 4,000,000 decode operations is ~5MB of dense object declarations, or ~4.5MB of dense object declarations with 10% alias expansion
// 4,000,000 decode operations is ~5MB of dense object declarations, or
// ~4.5MB of dense object declarations with 10% alias expansion
alias_ratio_range_high = 4000000
// alias_ratio_range is the range over which we scale allowed alias ratios
alias_ratio_range = float64(alias_ratio_range_high - alias_ratio_range_low)
)
@ -784,8 +788,7 @@ func (d *decoder) merge(n *node, out reflect.Value) {
case mappingNode:
d.unmarshal(n, out)
case aliasNode:
an, ok := d.doc.anchors[n.value]
if ok && an.kind != mappingNode {
if n.alias != nil && n.alias.kind != mappingNode {
failWantMap()
}
d.unmarshal(n, out)
@ -794,8 +797,7 @@ func (d *decoder) merge(n *node, out reflect.Value) {
for i := len(n.children) - 1; i >= 0; i-- {
ni := n.children[i]
if ni.kind == aliasNode {
an, ok := d.doc.anchors[ni.value]
if ok && an.kind != mappingNode {
if ni.alias != nil && ni.alias.kind != mappingNode {
failWantMap()
}
} else if ni.kind != mappingNode {

107
vendor/gopkg.in/yaml.v2/scannerc.go generated vendored

@ -626,31 +626,18 @@ func trace(args ...interface{}) func() {
func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool {
// While we need more tokens to fetch, do it.
for {
// Check if we really need to fetch more tokens.
need_more_tokens := false
if parser.tokens_head == len(parser.tokens) {
// Queue is empty.
need_more_tokens = true
} else {
// Check if any potential simple key may occupy the head position.
if !yaml_parser_stale_simple_keys(parser) {
if parser.tokens_head != len(parser.tokens) {
// If queue is non-empty, check if any potential simple key may
// occupy the head position.
head_tok_idx, ok := parser.simple_keys_by_tok[parser.tokens_parsed]
if !ok {
break
} else if valid, ok := yaml_simple_key_is_valid(parser, &parser.simple_keys[head_tok_idx]); !ok {
return false
}
for i := range parser.simple_keys {
simple_key := &parser.simple_keys[i]
if simple_key.possible && simple_key.token_number == parser.tokens_parsed {
need_more_tokens = true
break
}
} else if !valid {
break
}
}
// We are finished.
if !need_more_tokens {
break
}
// Fetch the next token.
if !yaml_parser_fetch_next_token(parser) {
return false
@ -678,11 +665,6 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool {
return false
}
// Remove obsolete potential simple keys.
if !yaml_parser_stale_simple_keys(parser) {
return false
}
// Check the indentation level against the current column.
if !yaml_parser_unroll_indent(parser, parser.mark.column) {
return false
@ -837,29 +819,30 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool {
"found character that cannot start any token")
}
// Check the list of potential simple keys and remove the positions that
// cannot contain simple keys anymore.
func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool {
// Check for a potential simple key for each flow level.
for i := range parser.simple_keys {
simple_key := &parser.simple_keys[i]
// The specification requires that a simple key
//
// - is limited to a single line,
// - is shorter than 1024 characters.
if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) {
// Check if the potential simple key to be removed is required.
if simple_key.required {
return yaml_parser_set_scanner_error(parser,
"while scanning a simple key", simple_key.mark,
"could not find expected ':'")
}
simple_key.possible = false
func yaml_simple_key_is_valid(parser *yaml_parser_t, simple_key *yaml_simple_key_t) (valid, ok bool) {
if !simple_key.possible {
return false, true
}
// The 1.2 specification says:
//
// "If the ? indicator is omitted, parsing needs to see past the
// implicit key to recognize it as such. To limit the amount of
// lookahead required, the “:” indicator must appear at most 1024
// Unicode characters beyond the start of the key. In addition, the key
// is restricted to a single line."
//
if simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index {
// Check if the potential simple key to be removed is required.
if simple_key.required {
return false, yaml_parser_set_scanner_error(parser,
"while scanning a simple key", simple_key.mark,
"could not find expected ':'")
}
simple_key.possible = false
return false, true
}
return true
return true, true
}
// Check if a simple key may start at the current position and add it if
@ -879,13 +862,14 @@ func yaml_parser_save_simple_key(parser *yaml_parser_t) bool {
possible: true,
required: required,
token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head),
mark: parser.mark,
}
simple_key.mark = parser.mark
if !yaml_parser_remove_simple_key(parser) {
return false
}
parser.simple_keys[len(parser.simple_keys)-1] = simple_key
parser.simple_keys_by_tok[simple_key.token_number] = len(parser.simple_keys) - 1
}
return true
}
@ -900,9 +884,10 @@ func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool {
"while scanning a simple key", parser.simple_keys[i].mark,
"could not find expected ':'")
}
// Remove the key from the stack.
parser.simple_keys[i].possible = false
delete(parser.simple_keys_by_tok, parser.simple_keys[i].token_number)
}
// Remove the key from the stack.
parser.simple_keys[i].possible = false
return true
}
@ -912,7 +897,12 @@ const max_flow_level = 10000
// Increase the flow level and resize the simple key list if needed.
func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool {
// Reset the simple key on the next level.
parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{})
parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{
possible: false,
required: false,
token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head),
mark: parser.mark,
})
// Increase the flow level.
parser.flow_level++
@ -928,7 +918,9 @@ func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool {
func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool {
if parser.flow_level > 0 {
parser.flow_level--
parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1]
last := len(parser.simple_keys) - 1
delete(parser.simple_keys_by_tok, parser.simple_keys[last].token_number)
parser.simple_keys = parser.simple_keys[:last]
}
return true
}
@ -1005,6 +997,8 @@ func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool {
// Initialize the simple key stack.
parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{})
parser.simple_keys_by_tok = make(map[int]int)
// A simple key is allowed at the beginning of the stream.
parser.simple_key_allowed = true
@ -1286,7 +1280,11 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool {
simple_key := &parser.simple_keys[len(parser.simple_keys)-1]
// Have we found a simple key?
if simple_key.possible {
if valid, ok := yaml_simple_key_is_valid(parser, simple_key); !ok {
return false
} else if valid {
// Create the KEY token and insert it into the queue.
token := yaml_token_t{
typ: yaml_KEY_TOKEN,
@ -1304,6 +1302,7 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool {
// Remove the simple key.
simple_key.possible = false
delete(parser.simple_keys_by_tok, simple_key.token_number)
// A simple key cannot follow another simple key.
parser.simple_key_allowed = false

2
vendor/gopkg.in/yaml.v2/yaml.go generated vendored

@ -89,7 +89,7 @@ func UnmarshalStrict(in []byte, out interface{}) (err error) {
return unmarshal(in, out, true)
}
// A Decorder reads and decodes YAML values from an input stream.
// A Decoder reads and decodes YAML values from an input stream.
type Decoder struct {
strict bool
parser *parser

1
vendor/gopkg.in/yaml.v2/yamlh.go generated vendored

@ -579,6 +579,7 @@ type yaml_parser_t struct {
simple_key_allowed bool // May a simple key occur at the current position?
simple_keys []yaml_simple_key_t // The stack of simple keys.
simple_keys_by_tok map[int]int // possible simple_key indexes indexed by token_number
// Parser stuff

22
vendor/modules.txt vendored

@ -205,6 +205,16 @@ github.com/gliderlabs/ssh
# github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a
## explicit
github.com/glycerine/go-unsnap-stream
# github.com/go-enry/go-enry/v2 v2.3.0
## explicit
github.com/go-enry/go-enry/v2
github.com/go-enry/go-enry/v2/data
github.com/go-enry/go-enry/v2/data/rule
github.com/go-enry/go-enry/v2/internal/tokenizer
github.com/go-enry/go-enry/v2/internal/tokenizer/flex
github.com/go-enry/go-enry/v2/regex
# github.com/go-enry/go-oniguruma v1.2.0
github.com/go-enry/go-oniguruma
# github.com/go-git/gcfg v1.5.0
github.com/go-git/gcfg
github.com/go-git/gcfg/scanner
@ -578,16 +588,6 @@ github.com/spf13/jwalterweatherman
github.com/spf13/pflag
# github.com/spf13/viper v1.4.0
github.com/spf13/viper
# github.com/src-d/enry/v2 v2.1.0
## explicit
github.com/src-d/enry/v2
github.com/src-d/enry/v2/data
github.com/src-d/enry/v2/data/rule
github.com/src-d/enry/v2/internal/tokenizer
github.com/src-d/enry/v2/internal/tokenizer/flex
github.com/src-d/enry/v2/regex
# github.com/src-d/go-oniguruma v1.1.0
github.com/src-d/go-oniguruma
# github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2
## explicit
github.com/steveyen/gtreap
@ -803,7 +803,7 @@ gopkg.in/testfixtures.v2
gopkg.in/toqueteos/substring.v1
# gopkg.in/warnings.v0 v0.1.2
gopkg.in/warnings.v0
# gopkg.in/yaml.v2 v2.2.4
# gopkg.in/yaml.v2 v2.2.8
gopkg.in/yaml.v2
# mvdan.cc/xurls/v2 v2.1.0
## explicit

Loading…
Cancel
Save