Abusing text/template for data transformation
4 likes1,928 views
The document discusses methods to gain visibility and manage open source projects, exemplified by a GitHub pull request related to adding a maintainer. It highlights relevant data transformation techniques using templates, including a TOML configuration approach. Furthermore, it touches upon user-defined functions and demonstrates how to collect and utilize project data effectively.
![1 {
2 "url": "https://api.github.com/repos/docker/docker/pulls/16603",
3 "id": 46083503,
4 "html_url": "https://github.com/docker/docker/pull/16603",
5 "diff_url": "https://github.com/docker/docker/pull/16603.diff",
6 "patch_url": "https://github.com/docker/docker/pull/16603.patch",
7 "issue_url": "https://api.github.com/repos/docker/docker/issues/16603",
8 "number": 16603,
9 "state": "closed",
10 "locked": false,
11 "title": "Add @vdemeester to MAINTAINERS",
12 "user": {
13 "login": "icecrime",
14 "id": 1564054,
15 "avatar_url": "https://avatars.githubusercontent.com/u/1564054?v=3",
16 "gravatar_id": "",
17 "url": "https://api.github.com/users/icecrime",
18 "html_url": "https://github.com/icecrime",
19 "followers_url": "https://api.github.com/users/icecrime/followers",
20 "following_url": "https://api.github.com/users/icecrime/following{/other_user}",
21 "gists_url": "https://api.github.com/users/icecrime/gists{/gist_id}",
22 "starred_url": "https://api.github.com/users/icecrime/starred{/owner}{/repo}",
23 "subscriptions_url": "https://api.github.com/users/icecrime/subscriptions",
24 "organizations_url": "https://api.github.com/users/icecrime/orgs",
25 "repos_url": "https://api.github.com/users/icecrime/repos",
26 "events_url": "https://api.github.com/users/icecrime/events{/privacy}",
27 "received_events_url": "https://api.github.com/users/icecrime/received_events",
28 "type": "User",
29 "site_admin": false
30 },
31 "body": ":tada:",
32 "created_at": "2015-09-26T15:16:23Z",
33 "updated_at": "2015-09-27T21:02:55Z",
34 "closed_at": "2015-09-27T19:14:39Z",
35 "merged_at": "2015-09-27T19:14:39Z",
36 "merge_commit_sha": "7fae194c5b4c694dc45a385866207df6bea57e61",
~~ ...
320 }
{
"author": {
"login": "icecrime",
"company": "Docker",
"is_maintainer": true
},
"body": ":tada:",
"closed_at": "2015-09-27T19:14:39Z",
"created_at": "2015-09-26T15:16:23Z",
"labels": ["status/4-merge"],
"merged": true,
"ms": "1.9.0",
"number": 16603,
"state": "closed",
"title": "Add @vdemeester...",
}](https://image.slidesharecdn.com/arnaudporterie-ligthningtalk-dotgo2015-151109133104-lva1-app6892/85/Abusing-text-template-for-data-transformation-5-320.jpg)
![{
"author": {
"login": "icecrime",
"company": "Docker",
"is_maintainer": true
},
"body": ":tada:",
"closed_at": "2015-09-27T19:14:39Z",
"created_at": "2015-09-26T15:16:23Z",
"labels": ["status/4-merge"],
"merged": true,
"ms": "1.9.0",
"number": 16603,
"state": "closed",
"title": "Add @vdemeester...",
}
[transformations.pull_request]
author = "{{ user_data .user.login }}"
body = "{{ .body }}"
closed_at = "{{ .closed_at }}"
created_at = "{{ .created_at }}"
labels = "{{ range .labels }}{{ .name }}{{ end }}"
merged = "{{ .merged }}"
ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}"
number = "{{ .number }}"
state = "{{ .state }}"
title = "{{ .title }}"](https://image.slidesharecdn.com/arnaudporterie-ligthningtalk-dotgo2015-151109133104-lva1-app6892/85/Abusing-text-template-for-data-transformation-8-320.jpg)
![[transformations.pull_request]
author = "{{ user_data .user.login }}"
body = "{{ .body }}"
closed_at = "{{ .closed_at }}"
created_at = "{{ .created_at }}"
labels = "{{ range .labels }}{{ .name }}{{ end }}"
merged = "{{ .merged }}"
ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}"
number = "{{ .number }}"
state = "{{ .state }}"
title = "{{ .title }}"
{
"author": {
"login": "icecrime",
"company": "Docker",
"is_maintainer": true
},
"body": ":tada:",
"closed_at": "2015-09-27T19:14:39Z",
"created_at": "2015-09-26T15:16:23Z",
"labels": ["status/4-merge"],
"merged": true,
"ms": "1.9.0",
"number": 16603,
"state": "closed",
"title": "Add @vdemeester...",
}
Simple mapping](https://image.slidesharecdn.com/arnaudporterie-ligthningtalk-dotgo2015-151109133104-lva1-app6892/85/Abusing-text-template-for-data-transformation-9-320.jpg)
![[transformations.pull_request]
author = "{{ user_data .user.login }}"
body = "{{ .body }}"
closed_at = "{{ .closed_at }}"
created_at = "{{ .created_at }}"
labels = "{{ range .labels }}{{ .name }}{{ end }}"
merged = "{{ .merged }}"
ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}"
number = "{{ .number }}"
state = "{{ .state }}"
title = "{{ .title }}"
{
"author": {
"login": "icecrime",
"company": "Docker",
"is_maintainer": true
},
"body": ":tada:",
"closed_at": "2015-09-27T19:14:39Z",
"created_at": "2015-09-26T15:16:23Z",
"labels": ["status/4-merge"],
"merged": true,
"ms": "1.9.0",
"number": 16603,
"state": "closed",
"title": "Add @vdemeester...",
}
Tests](https://image.slidesharecdn.com/arnaudporterie-ligthningtalk-dotgo2015-151109133104-lva1-app6892/85/Abusing-text-template-for-data-transformation-10-320.jpg)
![[transformations.pull_request]
author = "{{ user_data .user.login }}"
body = "{{ .body }}"
closed_at = "{{ .closed_at }}"
created_at = "{{ .created_at }}"
labels = "{{ range .labels }}{{ .name }}{{ end }}"
merged = "{{ .merged }}"
ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}"
number = "{{ .number }}"
state = "{{ .state }}"
title = "{{ .title }}"
{
"author": {
"login": "icecrime",
"company": "Docker",
"is_maintainer": true
},
"body": ":tada:",
"closed_at": "2015-09-27T19:14:39Z",
"created_at": "2015-09-26T15:16:23Z",
"labels": ["status/4-merge"],
"merged": true,
"ms": "1.9.0",
"number": 16603,
"state": "closed",
"title": "Add @vdemeester...",
}
Loops](https://image.slidesharecdn.com/arnaudporterie-ligthningtalk-dotgo2015-151109133104-lva1-app6892/85/Abusing-text-template-for-data-transformation-11-320.jpg)
![[transformations.pull_request]
author = "{{ user_data .user.login }}"
body = "{{ .body }}"
closed_at = "{{ .closed_at }}"
created_at = "{{ .created_at }}"
labels = "{{ range .labels }}{{ .name }}{{ end }}"
merged = "{{ .merged }}"
ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}"
number = "{{ .number }}"
state = "{{ .state }}"
title = "{{ .title }}"
{
"author": {
"login": "icecrime",
"company": "Docker",
"is_maintainer": true
},
"body": ":tada:",
"closed_at": "2015-09-27T19:14:39Z",
"created_at": "2015-09-26T15:16:23Z",
"labels": ["status/4-merge"],
"merged": true,
"ms": "1.9.0",
"number": 16603,
"state": "closed",
"title": "Add @vdemeester...",
}
User defined functions](https://image.slidesharecdn.com/arnaudporterie-ligthningtalk-dotgo2015-151109133104-lva1-app6892/85/Abusing-text-template-for-data-transformation-12-320.jpg)
Abusing text/template for data transformation
- 1. Abusing text/template Arnaud Porterie - @icecrime - dotGo 2015
- 2. How can I get visibility into my open source projects?
- 4. Filter Rename Enrich Transform 1 { 2 "url": "https://api.github.com/repos/docker/docker/pulls/16603", 3 "id": 46083503, 4 "html_url": "https://github.com/docker/docker/pull/16603", 5 "diff_url": "https://github.com/docker/docker/pull/16603.diff", 6 "patch_url": "https://github.com/docker/docker/pull/16603.patch", 7 "issue_url": "https://api.github.com/repos/docker/docker/issues/16603", 8 "number": 16603, 9 "state": "closed", 10 "locked": false, 11 "title": "Add @vdemeester to MAINTAINERS", 12 "user": { 13 "login": "icecrime", 14 "id": 1564054, 15 "avatar_url": "https://avatars.githubusercontent.com/u/1564054?v=3", 16 "gravatar_id": "", 17 "url": "https://api.github.com/users/icecrime", 18 "html_url": "https://github.com/icecrime", 19 "followers_url": "https://api.github.com/users/icecrime/followers", 20 "following_url": "https://api.github.com/users/icecrime/following{/other_user}", 21 "gists_url": "https://api.github.com/users/icecrime/gists{/gist_id}", 22 "starred_url": "https://api.github.com/users/icecrime/starred{/owner}{/repo}", 23 "subscriptions_url": "https://api.github.com/users/icecrime/subscriptions", 24 "organizations_url": "https://api.github.com/users/icecrime/orgs", 25 "repos_url": "https://api.github.com/users/icecrime/repos", 26 "events_url": "https://api.github.com/users/icecrime/events{/privacy}", 27 "received_events_url": "https://api.github.com/users/icecrime/received_events", 28 "type": "User", 29 "site_admin": false 30 }, 31 "body": ":tada:", 32 "created_at": "2015-09-26T15:16:23Z", 33 "updated_at": "2015-09-27T21:02:55Z", 34 "closed_at": "2015-09-27T19:14:39Z", 35 "merged_at": "2015-09-27T19:14:39Z", 36 "merge_commit_sha": "7fae194c5b4c694dc45a385866207df6bea57e61", ~~ ... 320 }
- 5. 1 { 2 "url": "https://api.github.com/repos/docker/docker/pulls/16603", 3 "id": 46083503, 4 "html_url": "https://github.com/docker/docker/pull/16603", 5 "diff_url": "https://github.com/docker/docker/pull/16603.diff", 6 "patch_url": "https://github.com/docker/docker/pull/16603.patch", 7 "issue_url": "https://api.github.com/repos/docker/docker/issues/16603", 8 "number": 16603, 9 "state": "closed", 10 "locked": false, 11 "title": "Add @vdemeester to MAINTAINERS", 12 "user": { 13 "login": "icecrime", 14 "id": 1564054, 15 "avatar_url": "https://avatars.githubusercontent.com/u/1564054?v=3", 16 "gravatar_id": "", 17 "url": "https://api.github.com/users/icecrime", 18 "html_url": "https://github.com/icecrime", 19 "followers_url": "https://api.github.com/users/icecrime/followers", 20 "following_url": "https://api.github.com/users/icecrime/following{/other_user}", 21 "gists_url": "https://api.github.com/users/icecrime/gists{/gist_id}", 22 "starred_url": "https://api.github.com/users/icecrime/starred{/owner}{/repo}", 23 "subscriptions_url": "https://api.github.com/users/icecrime/subscriptions", 24 "organizations_url": "https://api.github.com/users/icecrime/orgs", 25 "repos_url": "https://api.github.com/users/icecrime/repos", 26 "events_url": "https://api.github.com/users/icecrime/events{/privacy}", 27 "received_events_url": "https://api.github.com/users/icecrime/received_events", 28 "type": "User", 29 "site_admin": false 30 }, 31 "body": ":tada:", 32 "created_at": "2015-09-26T15:16:23Z", 33 "updated_at": "2015-09-27T21:02:55Z", 34 "closed_at": "2015-09-27T19:14:39Z", 35 "merged_at": "2015-09-27T19:14:39Z", 36 "merge_commit_sha": "7fae194c5b4c694dc45a385866207df6bea57e61", ~~ ... 320 } { "author": { "login": "icecrime", "company": "Docker", "is_maintainer": true }, "body": ":tada:", "closed_at": "2015-09-27T19:14:39Z", "created_at": "2015-09-26T15:16:23Z", "labels": ["status/4-merge"], "merged": true, "ms": "1.9.0", "number": 16603, "state": "closed", "title": "Add @vdemeester...", }
- 6. "Hello dotGo!" package main import ( "os" "text/template" ) type Foo struct { Bar string } type Something struct { Foo Foo } const text = "Hello {{ .Foo.Bar }}!" func main() { obj := Something{ Foo{ Bar: "dotGo", }, } t, _ := template.New("").Parse(text) t.Execute(os.Stdout, obj) }
- 7. What if we... ● Fork text/template ● Substitute fmt.Fprint for a return ● Use the template syntax as a DSL for data transformation ● Describe the model in a TOML configuration file
- 8. { "author": { "login": "icecrime", "company": "Docker", "is_maintainer": true }, "body": ":tada:", "closed_at": "2015-09-27T19:14:39Z", "created_at": "2015-09-26T15:16:23Z", "labels": ["status/4-merge"], "merged": true, "ms": "1.9.0", "number": 16603, "state": "closed", "title": "Add @vdemeester...", } [transformations.pull_request] author = "{{ user_data .user.login }}" body = "{{ .body }}" closed_at = "{{ .closed_at }}" created_at = "{{ .created_at }}" labels = "{{ range .labels }}{{ .name }}{{ end }}" merged = "{{ .merged }}" ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}" number = "{{ .number }}" state = "{{ .state }}" title = "{{ .title }}"
- 9. [transformations.pull_request] author = "{{ user_data .user.login }}" body = "{{ .body }}" closed_at = "{{ .closed_at }}" created_at = "{{ .created_at }}" labels = "{{ range .labels }}{{ .name }}{{ end }}" merged = "{{ .merged }}" ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}" number = "{{ .number }}" state = "{{ .state }}" title = "{{ .title }}" { "author": { "login": "icecrime", "company": "Docker", "is_maintainer": true }, "body": ":tada:", "closed_at": "2015-09-27T19:14:39Z", "created_at": "2015-09-26T15:16:23Z", "labels": ["status/4-merge"], "merged": true, "ms": "1.9.0", "number": 16603, "state": "closed", "title": "Add @vdemeester...", } Simple mapping
- 10. [transformations.pull_request] author = "{{ user_data .user.login }}" body = "{{ .body }}" closed_at = "{{ .closed_at }}" created_at = "{{ .created_at }}" labels = "{{ range .labels }}{{ .name }}{{ end }}" merged = "{{ .merged }}" ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}" number = "{{ .number }}" state = "{{ .state }}" title = "{{ .title }}" { "author": { "login": "icecrime", "company": "Docker", "is_maintainer": true }, "body": ":tada:", "closed_at": "2015-09-27T19:14:39Z", "created_at": "2015-09-26T15:16:23Z", "labels": ["status/4-merge"], "merged": true, "ms": "1.9.0", "number": 16603, "state": "closed", "title": "Add @vdemeester...", } Tests
- 11. [transformations.pull_request] author = "{{ user_data .user.login }}" body = "{{ .body }}" closed_at = "{{ .closed_at }}" created_at = "{{ .created_at }}" labels = "{{ range .labels }}{{ .name }}{{ end }}" merged = "{{ .merged }}" ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}" number = "{{ .number }}" state = "{{ .state }}" title = "{{ .title }}" { "author": { "login": "icecrime", "company": "Docker", "is_maintainer": true }, "body": ":tada:", "closed_at": "2015-09-27T19:14:39Z", "created_at": "2015-09-26T15:16:23Z", "labels": ["status/4-merge"], "merged": true, "ms": "1.9.0", "number": 16603, "state": "closed", "title": "Add @vdemeester...", } Loops
- 12. [transformations.pull_request] author = "{{ user_data .user.login }}" body = "{{ .body }}" closed_at = "{{ .closed_at }}" created_at = "{{ .created_at }}" labels = "{{ range .labels }}{{ .name }}{{ end }}" merged = "{{ .merged }}" ms = "{{ if $m := .milestone }}{{ $m.title }}{{ end }}" number = "{{ .number }}" state = "{{ .state }}" title = "{{ .title }}" { "author": { "login": "icecrime", "company": "Docker", "is_maintainer": true }, "body": ":tada:", "closed_at": "2015-09-27T19:14:39Z", "created_at": "2015-09-26T15:16:23Z", "labels": ["status/4-merge"], "merged": true, "ms": "1.9.0", "number": 16603, "state": "closed", "title": "Add @vdemeester...", } User defined functions
- 13. TOML JSON
- 15. Profit! Thank you Arnaud Porterie - @icecrime /icecrime/vossibility-collector