Import export (#194)

* start with export/import code

* messing about with decoding/encoding

* some more fiddling

* stuff is WORKING

* working pretty alright!

* go fmt

* fix up tests, add docs

* start backup/restore doc

* tweaks

* credits

* update advancedVisibility settings

* update bun library -> v1.0.4

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update oauth library -> v4.3.1-SSB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* handle oauth token scope, fix user.SigninCount + token.UserID

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update oauth library --> v4.3.2-SSB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update sqlite library -> v1.13.0

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* review changes

* start with export/import code

* messing about with decoding/encoding

* some more fiddling

* stuff is WORKING

* working pretty alright!

* go fmt

* fix up tests, add docs

* start backup/restore doc

* tweaks

* credits

* update advancedVisibility settings

* review changes

Co-authored-by: kim (grufwub) <grufwub@gmail.com>
Co-authored-by: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>

README.md

@@ -134,10 +134,11 @@ The following libraries and frameworks are used by GoToSocial, with gratitude
 * [gorilla/websocket](https://github.com/gorilla/websocket); Websocket connectivity. [BSD-2-Clause License](https://spdx.org/licenses/BSD-2-Clause.html).
 * [h2non/filetype](https://github.com/h2non/filetype); filetype checking. [MIT License](https://spdx.org/licenses/MIT.html).
 * [jackc/pgx](https://github.com/jackc/pgx); Postgres driver. [MIT License](https://spdx.org/licenses/MIT.html).
+* [microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday); HTML user-input sanitization. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
+* [mitchellh/mapstructure](https://github.com/mitchellh/mapstructure); Go interface => struct parsing. [MIT License](https://spdx.org/licenses/MIT.html).
 * [modernc.org/sqlite](https://gitlab.com/cznic/sqlite); cgo-free port of SQLite. [Other License](https://gitlab.com/cznic/sqlite/-/blob/master/LICENSE).
   * [modernc.org/ccgo](https://gitlab.com/cznic/ccgo); c99 AST -> Go translater. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
   * [modernc.org/libc](https://gitlab.com/cznic/libc); C-runtime services. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
-* [microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday); HTML user-input sanitization. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
 * [mvdan/xurls](https://github.com/mvdan/xurls); URL parsing regular expressions. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html).
 * [nfnt/resize](https://github.com/nfnt/resize); convenient image resizing. [ISC License](https://spdx.org/licenses/ISC.html).
 * [oklog/ulid](https://github.com/oklog/ulid); sequential, database-friendly ID generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).

cmd/gotosocial/admincommands.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/account"
+	"github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/trans"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/urfave/cli/v2"
 )
@@ -39,16 +40,19 @@ func adminCommands() []*cli.Command {
 							Usage: "create a new account",
 							Flags: []cli.Flag{
 								&cli.StringFlag{
-									Name:  config.UsernameFlag,
+									Name:     config.UsernameFlag,
-									Usage: config.UsernameUsage,
+									Usage:    config.UsernameUsage,
+									Required: true,
 								},
 								&cli.StringFlag{
-									Name:  config.EmailFlag,
+									Name:     config.EmailFlag,
-									Usage: config.EmailUsage,
+									Usage:    config.EmailUsage,
+									Required: true,
 								},
 								&cli.StringFlag{
-									Name:  config.PasswordFlag,
+									Name:     config.PasswordFlag,
-									Usage: config.PasswordUsage,
+									Usage:    config.PasswordUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -60,8 +64,9 @@ func adminCommands() []*cli.Command {
 							Usage: "confirm an existing account manually, thereby skipping email confirmation",
 							Flags: []cli.Flag{
 								&cli.StringFlag{
-									Name:  config.UsernameFlag,
+									Name:     config.UsernameFlag,
-									Usage: config.UsernameUsage,
+									Usage:    config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -73,8 +78,9 @@ func adminCommands() []*cli.Command {
 							Usage: "promote an account to admin",
 							Flags: []cli.Flag{
 								&cli.StringFlag{
-									Name:  config.UsernameFlag,
+									Name:     config.UsernameFlag,
-									Usage: config.UsernameUsage,
+									Usage:    config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -86,8 +92,9 @@ func adminCommands() []*cli.Command {
 							Usage: "demote an account from admin to normal user",
 							Flags: []cli.Flag{
 								&cli.StringFlag{
-									Name:  config.UsernameFlag,
+									Name:     config.UsernameFlag,
-									Usage: config.UsernameUsage,
+									Usage:    config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -99,8 +106,9 @@ func adminCommands() []*cli.Command {
 							Usage: "prevent an account from signing in or posting etc, but don't delete anything",
 							Flags: []cli.Flag{
 								&cli.StringFlag{
-									Name:  config.UsernameFlag,
+									Name:     config.UsernameFlag,
-									Usage: config.UsernameUsage,
+									Usage:    config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -112,8 +120,9 @@ func adminCommands() []*cli.Command {
 							Usage: "completely remove an account and all of its posts, media, etc",
 							Flags: []cli.Flag{
 								&cli.StringFlag{
-									Name:  config.UsernameFlag,
+									Name:     config.UsernameFlag,
-									Usage: config.UsernameUsage,
+									Usage:    config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -125,12 +134,14 @@ func adminCommands() []*cli.Command {
 							Usage: "set a new password for the given account",
 							Flags: []cli.Flag{
 								&cli.StringFlag{
-									Name:  config.UsernameFlag,
+									Name:     config.UsernameFlag,
-									Usage: config.UsernameUsage,
+									Usage:    config.UsernameUsage,
+									Required: true,
 								},
 								&cli.StringFlag{
-									Name:  config.PasswordFlag,
+									Name:     config.PasswordFlag,
-									Usage: config.PasswordUsage,
+									Usage:    config.PasswordUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -139,6 +150,34 @@ func adminCommands() []*cli.Command {
 						},
 					},
 				},
+				{
+					Name:  "export",
+					Usage: "export data from the database to file at the given path",
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:     config.TransPathFlag,
+							Usage:    config.TransPathUsage,
+							Required: true,
+						},
+					},
+					Action: func(c *cli.Context) error {
+						return runAction(c, trans.Export)
+					},
+				},
+				{
+					Name:  "import",
+					Usage: "import data from a file into the database",
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:     config.TransPathFlag,
+							Usage:    config.TransPathUsage,
+							Required: true,
+						},
+					},
+					Action: func(c *cli.Context) error {
+						return runAction(c, trans.Import)
+					},
+				},
 			},
 		},
 	}

docs/admin/backup_and_restore.md

@@ -0,0 +1,102 @@
+# Backup and Restore
+
+In certain conditions, it may be desirable to be able to back up a GoToSocial instance, and then to restore it again later, or just save the backup somewhere.
+
+Some potential scenarios:
+
+* You want to close down your instance but you might create it again later and you don't want to break federation.
+* You need to migrate to a different database for some reason (Postgres => SQLite or vice versa).
+* You want to keep regular backups of your data just in case something happens.
+* You want to migrate from GoToSocial to a different Fediverse server, or from a different Fediverse server to GoToSocial.
+* You're about to hack around on your instance and you want to make a quick backup so you don't lose everything if you mess up.
+
+There are a few different ways of doing this, most of which require some technical knowledge.
+
+## Image your disk
+
+If you're running GoToSocial on a VPS (a remote machine in the cloud), arguably the easiest way to preserve all of your database entries and media is to image the disk attached to the VPS. This will preserve the whole disk. Many VPS providers offer the option of automatically creating backups on a timer, so you'll always be able to restore if your data is lost.
+
+Advantages:
+
+* Relatively easy to do.
+* Easy to automate (depending on your vps).
+* Keep complete media + database entries.
+
+Disadvantages:
+
+* Can cost extra depending on your VPS.
+* Will probably also preserve stuff you don't need, from other programs running on the same machine.
+* Vendor lock-in, difficult to move the data around.
+
+## Back up your database files
+
+Regardless of whether you're using Postgres or SQLite as your GoToSocial database, it's possible to simply back up the database files directly by using something like [rclone](https://rclone.org/), or following best practices for [backing up Postgres data](https://www.postgresql.org/docs/9.1/backup.html) or [SQLite data](https://sqlite.org/backup.html).
+
+Advantages:
+
+* Backups are relatively portable - you can move data from one machine to another.
+* Well-documented procedure with a lot of guides and tooling available.
+* Lots of different ways of doing your backups, depending on what you need.
+
+Disadvantages:
+
+* Can be a bit fiddly to set up initially.
+* You need to figure out where to keep your backups.
+* Restoring from backups can be a pain.
+* Unless you back up media as well, references to media attachments in your db will be broken.
+
+## Use the GoToSocial CLI
+
+The GoToSocial CLI tool also provides commands for backing up and restoring data from your instance, which will preserve the *bare-minimum* necessary data to backup and restore your instance, without breaking federation with other instances.
+
+What will be **kept**:
+
+* All local account entries, including private and public keys.
+* Followed/following remote accounts, including public keys.
+* Follows/follow requests.
+* Domain blocks.
+* Account blocks.
+* Account suspensions.
+* User + password entries, email addresses.
+
+What will be **dropped**:
+
+* All statuses.
+* Media.
+* Faves.
+* Bookmarks.
+* Pins.
+* Applications.
+* Tokens.
+
+The backup file produced will be in the form of a line-separated series of JSON objects (not a JSON array!). For example:
+
+```json
+{"type":"account","id":"01F8MH5NBDF2MV7CTC4Q5128HF","createdAt":"2021-08-31T12:00:53.985645Z","username":"1happyturtle","locked":true,"language":"en","uri":"http://localhost:8080/users/1happyturtle","url":"http://localhost:8080/@1happyturtle","inboxURI":"http://localhost:8080/users/1happyturtle/inbox","outboxURI":"http://localhost:8080/users/1happyturtle/outbox","followingUri":"http://localhost:8080/users/1happyturtle/following","followersUri":"http://localhost:8080/users/1happyturtle/followers","featuredCollectionUri":"http://localhost:8080/users/1happyturtle/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjz\nausfsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLz\neUPxdfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFx\njUz9l0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJY\nfKhKn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq\n79WbhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQABAoIBAGF+MxHjD15VV2NY\nKKb1GjMx98i1Xx6TijgoA+zmfha4LGu35e79Lql+0LXFp0zEpa6lAQsMQQhgd0OD\nmKKmSk+pxAvskJ4FxrhIf/yBFA4RMrj5OCaAOocRtdsOJ8n5UtFBrNAF0tzMY9q/\nkgzoq97aVF1mV9iFxaeBx6zT8ozSdqBq1PK/3w1dVg89S5tfKYc7Q0lQ00SfsTnd\niTDClKyqurebo9Pt6M7gXavgg3tvBlmwwr6XHs34Leng3oiN9mW8DVzaBMPzn+rE\nxF2eqs3v9vVpj8es88OwCh5P+ff8vJYvhu7Fcr/bJ8BItBQwfb8QBDATg/MXU2BI\n2ssW6AECgYEA4wmIyYGeu9+hzDa/J3Vh8GnlVNUCohHcChQdOsWsFXUgpVlUIHrX\neKHn42vD4Rzy52/YzJts4NkZTM9sL+kEXIEcpMG/S9xIIud7U0m/hMSAlmnJK/9j\niEXws3o4jo0E77jnRcBdIjpG4K5Eekm0DSR3SFhtZfEdN2DWPvu7K98CgYEA5tER\n/qJwFMc51AobMU87ZjXON7hI2U1WY/pVF62jSl0IcSsnj2riEKWLrs+GRG+HUg+U\naFSqAHcxaVHA0h0AYR8RopAhDdVKh0kvB8biLo+IEzNjPv2vyn0yRN5YSfXdGzyJ\nUjVU6kWdQOwmzy86nHgFaqEx7eofHIaGZzJK/AECgYEAu2VNQHX63TuzQuoVUa5z\nzoq5vhGsALYZF0CO98ndRkDNV22qIL0ESQ/qZS64GYFZhWouWoQXlGfdmCbFN65v\n6SKwz9UT3rvN1vGWO6Ltr9q6AG0EnYpJT1vbV2kUcaU4Y94NFue2d9/+TMnKv91B\n/m8Q/efvNGuWH/WQIaCKV6UCgYBz89WhYMMDfS4M2mLcu5vwddk53qciGxrqMMjs\nkzsz0Va7W12NS7lzeWaZlAE0gf6t98urOdUJVNeKvBoss4sMP0phqxwf0eWV3ur0\ncjIQB+TpGGikLVdRVuGY/UXHKe9AjoHBva8B3aTpB3lbnbNJBXZbIc1uYq3sa5w7\nXWWUAQKBgH3yW73RRpQNcc9hTUssomUsnQQgHxpfWx5tNxqod36Ytd9EKBh3NqUZ\nvPcH6gdh7mcnNaVNTtQOHLHsbPfBK/pqvb3MAsdlokJcQz8MQJ9SGBBPY6PaGw8z\nq/ambaQykER6dwlXTIlU20uXY0bttOL/iYjKmgo3vA66qfzS6nsg\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjzausf\nsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLzeUPx\ndfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFxjUz9\nl0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJYfKhK\nn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq79Wb\nhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/1happyturtle#main-key"}
+{"type":"account","id":"01F8MH0BBE4FHXPH513MBVFHB0","createdAt":"2021-09-08T10:00:53.985634Z","username":"weed_lord420","locked":true,"language":"en","uri":"http://localhost:8080/users/weed_lord420","url":"http://localhost:8080/@weed_lord420","inboxURI":"http://localhost:8080/users/weed_lord420/inbox","outboxURI":"http://localhost:8080/users/weed_lord420/outbox","followingUri":"http://localhost:8080/users/weed_lord420/following","followersUri":"http://localhost:8080/users/weed_lord420/followers","featuredCollectionUri":"http://localhost:8080/users/weed_lord420/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0b\nMIyLRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//P\nceYpo5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4\nus6VxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+\nfNyYVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPc\nqwtx0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQABAoIBAEAA4GHNS4k+Ke4j\nx4J0XkUjV5UbuPY0pSpSDjOJHOJmUfLcg85Ds9mYYO6zxwOaqmrC42ieclI5rh84\nTWQUqX9+VAk1J9UKeE4xZ1SSBtnZ3rK9PjrERZ+dmQ0dATaCuEO5Wwgu7Trk++Bg\nIqy8WNGZL94v9tfwALp1jTXW9AvmQoNdCFBP62vcmYW4YLjnggxLCFTA8YKfdePa\nTuxxY6uLkeBbxzWpbRU2+bmlxd5OnCkiRSMHIX+6JdtCu2JdWpUTCnWrFi2n1TZz\nZQx9z5rvowK1O785jGMFum5vBWpjIU8sJcXmPjGMU25zzmrhzfmkJsTXER3CXoUo\nSqSPqgECgYEA78OR7bY5KKQQ7Lyz6dru4Fct5P/OXTQoOg5aS7TKb95LVWj+TANn\n5djwIbLmAUV30z0Id9VgiZOL0Hny8+3VV9eU088Z408pAy5WQrL3dB8tZLUJSq5c\n5k6X15/VjWOOZKppDxShzoV3mcohrnwVwkv4fhPFQQOJJBYz6xurWs0CgYEA3MDE\nsDMd9ahzO0dl62ynojkkA8ZTcn2UdyvLpGj9UxT5j9vWF3CfqitXgcpNiVSIbxqQ\nbo/pBch7c/2Xakv5zkdcrJj5/6gyr+m1/tK2o7+CjDaSE4SYwufXx+qkl03Zpyzt\nKdOi7Hz/b2tdjump7ECEDE45mG2ea8oSnPgXl0cCgYBkGGFzu/9g2B24t47ksmHH\nhp3CXIjqoDurARLxSCi7SzJoFc0ULtfRPSAC8YzUOwwrQ++lF4+V3+MexcqHy2Kl\nqXqYcn18SC/3BAE/Fzf3Yoyw3mNiqihefbEmc7PTsxxfKkVx5ksmzNGBgsFM9sCe\nvNigyeAvpCo8xogmPwbqgQKBgE34mIBTzcUzFmBdu5YH7r3RyPK8XkUWLhZZlbgg\njTmHMw6o61mkIgENBf+F4RUckoQLsfAbTIcKZPB3JcAZzcYaVpVwAv1V/3E671lu\nO6xivE2iCL50GzDcis7GBhSbHsF5kNsxMV6uV9qW5ZjQ13/m2b0u9BDuxwHzgdeH\nmW2JAoGAIUOYniuEwdygxWVnYatpr3NPjT3BOKoV5i9zkeJRu1hFpwQM6vQ4Ds5p\nGC5vbMKAv9Cwuw62e2HvqTun3+U2Y5Uived3XCpgM/50BFrFHCfuqXEnu1bEzk5z\n9mIhp8uXPxzC5N7tRQfb3/eU1IUcb6T6ksbr2P81z0j03J55erg=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0bMIyL\nRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//PceYp\no5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4us6V\nxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+fNyY\nVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPcqwtx\n0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/weed_lord420#main-key"}
+{"type":"account","id":"01F8MH17FWEB39HZJ76B6VXSKF","createdAt":"2021-09-05T10:00:53.985641Z","username":"admin","locked":true,"language":"en","uri":"http://localhost:8080/users/admin","url":"http://localhost:8080/@admin","inboxURI":"http://localhost:8080/users/admin/inbox","outboxURI":"http://localhost:8080/users/admin/outbox","followingUri":"http://localhost:8080/users/admin/following","followersUri":"http://localhost:8080/users/admin/followers","featuredCollectionUri":"http://localhost:8080/users/admin/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVq\nhujDhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLR\nBI97qD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wg\nfvtEjEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G\n8kQJDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/Bk\nRhhGp2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQABAoIBAGK0aIADOU4ffJDe\n7sveiih5Fc1PATwx/QIR2QkWM1SREdx6LYclcX44V8xDanAbE44p1SkHY/CsEtYy\nXnyoXnn2FwFDQrdveY7+I6PApOPLAcKWkyLltC+hbVdj92/6YGNrm7EA/a77wruH\nmwjiivLnTG2CLecNiXSl33DA9YU4Yz+2Tza3IpTdjt8c/dz/BKKaxaWV+i9ew5VR\nioo5v51B+J8PrneCM/p8LGiLV148Njr0JqV6eFy1JuzItYMYdc3Fp+YnMzsuMZEA\n1akMcoln/ucVJyOFnCn6jx47nIoPZLl1KxX3aRDRfvrejm6W4yAkkTmR5voSRqax\njPL3rI0CgYEA9Acu4TO8xJ3uGaUad0N9JTYQVSmtAaE/g+df9LGMSzoj8X95S4xE\nQsGPqNGDm2VWADJjK4P05twZ+LfsfSKQ86wbp4/gbgnXpqB1P5Lty/B7KxiTnNwt\nwb1WGWTCukxfUSL3PRyf8uylkrg72RxKiBx4zKO3WVSLWOZWrFtn0qMCgYEA0H2p\nJs9Nv20ADOOX5tQ7+ruS6/B/Fhyj5fhflSYCAtOW7aME7+zQKJyqSQZ4b2Aub3Tp\nGIaUbRIGzjHyuTultFFWvjU3H5aI/0g1G9WKaBhNkyTIYVmMKtYyhXNvouWing8x\noraWx8TTBP8Cdnnk+QgdR2fpug8cghKupp5wvO8CgYA1JFtRL7MsHjh73TimQExA\njkWARlMmx7bNQtXis8eZmk+5h8kiaqly4DQoz3eZn7fa0x5Fm7b5j3UYdPVLSvvG\nFPTwyKRXUk1kPA1MivK+NuCbwf5jao+MYW8emJLPf1JCmRq+dD1g6aglC3n9Dewt\nOAYWipCjI4Y1FfRKFJ3HgQKBgEAb47+DTyzln3ZXJYZdDHR06SCTuwBZnixAy2NZ\nZJTp6yb3UbVU5E0Yn2QFEVNuB9lN4b8g4tMHEACnazN6G+HugPXL9z9HUqjs0yfT\n6dNIZdIxJUyJ9IfXhYFzlYhJhE+F7IVUD9kttJV8tI0pvja1QAuM8Fm9+84jYIDr\nh08RAoGAMYbjKHbtejcHBwt1kIcSss0cDmlZbBleJo8tdmdg4ndf5GE9N4/EL7tq\nm2zYSfr7OVdnOwRhoO+xF/6d1L7+TR1wz+k2fuMsI71aM5Ocp1nYTutjIkBTcldZ\nZzvjOgZWng5icuRLQQiDSKG5uqazqL/xGXkijb4kp4WW6myWY3c=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVqhujD\nhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLRBI97\nqD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wgfvtE\njEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G8kQJ\nDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/BkRhhG\np2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/admin#main-key"}
+{"type":"account","id":"01F8MH1H7YV1Z7D2C8K2730QBF","createdAt":"2021-09-06T10:00:53.985643Z","username":"the_mighty_zork","locked":true,"language":"en","uri":"http://localhost:8080/users/the_mighty_zork","url":"http://localhost:8080/@the_mighty_zork","inboxURI":"http://localhost:8080/users/the_mighty_zork/inbox","outboxURI":"http://localhost:8080/users/the_mighty_zork/outbox","followingUri":"http://localhost:8080/users/the_mighty_zork/following","followersUri":"http://localhost:8080/users/the_mighty_zork/followers","featuredCollectionUri":"http://localhost:8080/users/the_mighty_zork/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss\n5mEA/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvC\nC9zt/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZ\nFHptEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1\ntMhsUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlq\nefr58l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQABAoIBAFa+UypbFG1cW2Tr\nNBxPm7ngOEtXl8MicV4dIVKh0TwOo13ZxtNFBbOj7jALmPn/9HrtmbkABPQHDL1U\n/nt9aNSAeTjpwH3RaD5vFX3n0g8n2zJBOZLxxzAjNi4RBLYj5uP1AiKkdvRlsJza\nuSFDkty2zMBqN9mLPHE+RePj5Qa6tjYfIQqQzu/+YnYMlXHoC2yHNKsvz6S5FhVj\nv5zATv2JlJQH3RSmhuPOah73iQnKCLzYYEAHleawKrCg/rZ3ht37Guvabeq7MqQN\nvi9pJdAA+RMxPsboHajskePjOTYJgKQSxEAMRTMfBR40aZxklxQL0EoBd1Y3CHXh\nfMg0xWECgYEA0ORrpJ1A2WNQwKcDDeBBsaJqWF4EraoFzYrugKZrAYEeVyuGD0zq\nARUaWkZTZ1f6wQ10i1WxAuKlBEds7QsLdZzLsA4um4JlBroCZiYfPnmTtb8op1LY\nFqeYTByvAmnfWWTuOI67GX9ruLg8tEGuz38kuQVSxYs51its3tScNPUCgYEAyRst\nwRbqpOqnwoRoS6pxv0Vpc3nUcfaVYwsg/qobJkiwAdlUYeE7alvEY926VW4cvU/X\nhy3L1punAqnyLI7uuqCefXEbNxO0Cebyy4Kv2Ye1uzl0OHsJczSNdfpNqfAIKwtN\nHLCYDGCsluQhz+I/5Pd0dT+JDPPW9hKS2HG7o+kCgYBqugn1VRLo/sEnbS02TbnC\n1ESZWY/yWsgUOEObH2vUnO+vgeFAt/9nBi0sqnm6d0z6jbFZ7zI9UycUhJm2ksoM\nEUxQay6M7ZZIVYkcP6X++YbqePyAYOdey8oYOR+BkC45MkQ0SVh2so+LFTaOsnBq\nO3+7uGiN3ZBzSESbpO0acQKBgQCONrsXZeZO82XpB4tdns3LbgGRWKEkajTgEnml\nvZNvck2NMSwb/5PttbFe0ei4CyMluPV4MamJPQ9Qse+BFR67OWR63uZY/4T8z6X4\nxpUmZnLcUFfgrRlUr+AtgvEy8HxGPDquxC7x6deC6RcEFEIM3/UqCOEZGMJ1x1Ky\n31LLKQKBgGCKwVgQ8+4JyHZFkie3YdHhxJDokgY+Opb0HNnoBY/lZ54UMCCJQPS2\n0XPSu651j/3adr3RQneU04gF6U2/D5JzFEV0kUsqZ4Zy2EEU0LU4ibus0gyomSpK\niWhU4QrC/M4ELxYZinlNu3ThPWNQ/PMNteVWfdgOcV7uUWl0ViFp\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss5mEA\n/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvCC9zt\n/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZFHpt\nEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1tMhs\nUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlqefr5\n8l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/the_mighty_zork#main-key"}
+{"type":"block","id":"01FEXXET6XXMF7G2V3ASZP3YQW","createdAt":"2021-09-08T09:00:53.965362Z","uri":"http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW","accountId":"01F8MH5NBDF2MV7CTC4Q5128HF","targetAccountId":"01F8MH5ZK5VRH73AKHQM6Y9VNX"}
+{"type":"account","id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","createdAt":"2021-08-31T12:00:53.985646Z","username":"foss_satan","domain":"fossbros-anonymous.io","locked":true,"language":"en","uri":"http://fossbros-anonymous.io/users/foss_satan","url":"http://fossbros-anonymous.io/@foss_satan","inboxURI":"http://fossbros-anonymous.io/users/foss_satan/inbox","outboxURI":"http://fossbros-anonymous.io/users/foss_satan/outbox","followingUri":"http://fossbros-anonymous.io/users/foss_satan/following","followersUri":"http://fossbros-anonymous.io/users/foss_satan/followers","featuredCollectionUri":"http://fossbros-anonymous.io/users/foss_satan/collections/featured","actorType":"Person","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEA2OyVgkaIL9VohXKYTh319j4OouHRX/8QC7piXj71k7q5RDzEyvis\nVZBc5/C1/crCpxt895i0Ai2CiXQx+dISV7s/JBhAGl8s7TQ8jLlMuptrI0+sdkBC\nlu8pU0qQmoeXVnlquOzNmqGufUxIDtLXLZDN17qf/7vWA23q4d0tG5KQhGGGKiVM\n61Ufvr9MmgPBSpyUvYMAulFlz1264L49aGWeVgOz3qUQzqtxjrP0kaIbeyt56miP\nKr5AqkRgSsXci+FAo6suxR5gzo9NgleNkbZWF9MQyKlawukPwZUDSh396vtNQMee\n/4mto7mAXw8iio0IacrYO3F7iyewXnmI/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://fossbros-anonymous.io/users/foss_satan/main-key"}
+{"type":"follow","id":"01F8PYDCE8XE23GRE5DPZJDZDP","createdAt":"2021-09-08T09:00:54.749465Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PYDCE8XE23GRE5DPZJDZDP","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH5NBDF2MV7CTC4Q5128HF"}
+{"type":"follow","id":"01F8PY8RHWRQZV038T4E8T9YK8","createdAt":"2021-09-06T12:00:54.749459Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PY8RHWRQZV038T4E8T9YK8","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH17FWEB39HZJ76B6VXSKF"}
+{"type":"domainBlock","id":"01FF22EQM7X8E3RX1XGPN7S87D","createdAt":"2021-09-08T10:00:53.968971Z","domain":"replyguys.com","createdByAccountID":"01F8MH17FWEB39HZJ76B6VXSKF","privateComment":"i blocked this domain because they keep replying with pushy + unwarranted linux advice","publicComment":"reply-guying to tech posts","obfuscate":false}
+{"type":"user","id":"01F8MGYG9E893WRHW0TAEXR8GJ","createdAt":"2021-09-08T10:00:53.97247Z","accountID":"01F8MH0BBE4FHXPH513MBVFHB0","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","locale":"en","lastEmailedAt":"0001-01-01T00:00:00Z","confirmationToken":"a5a280bd-34be-44a3-8330-a57eaf61b8dd","confirmationTokenSentAt":"2021-09-08T10:00:53.972472Z","unconfirmedEmail":"weed_lord420@example.org","moderator":false,"admin":false,"disabled":false,"approved":false}
+{"type":"user","id":"01F8MGWYWKVKS3VS8DV1AMYPGE","createdAt":"2021-09-05T10:00:53.972475Z","email":"admin@example.org","accountID":"01F8MH17FWEB39HZJ76B6VXSKF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:50:53.972477Z","lastSignInAt":"2021-09-08T08:00:53.972477Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:30:53.972478Z","confirmedAt":"2021-09-05T10:00:53.972478Z","moderator":true,"admin":true,"disabled":false,"approved":true}
+{"type":"user","id":"01F8MGVGPHQ2D3P3X0454H54Z5","createdAt":"2021-09-06T22:00:53.97248Z","email":"zork@example.org","accountID":"01F8MH1H7YV1Z7D2C8K2730QBF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972481Z","lastSignInAt":"2021-09-08T08:00:53.972481Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972482Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972483Z","confirmedAt":"2021-09-07T00:00:53.972482Z","moderator":false,"admin":false,"disabled":false,"approved":true}
+{"type":"user","id":"01F8MH1VYJAE00TVVGMM5JNJ8X","createdAt":"2021-09-06T22:00:53.972485Z","email":"tortle.dude@example.org","accountID":"01F8MH5NBDF2MV7CTC4Q5128HF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972485Z","lastSignInAt":"2021-09-08T08:00:53.972486Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972487Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972487Z","confirmedAt":"2021-09-07T00:00:53.972487Z","moderator":false,"admin":false,"disabled":false,"approved":true}
+{"type":"instance","id":"01BZDDRPAB8J645ABY31HHF68Y","createdAt":"2021-09-08T10:00:54.763912Z","domain":"localhost:8080","title":"localhost:8080","uri":"http://localhost:8080","reputation":0}
+```
+
+For information on how to use the commands to import/export, see [here](cli.md#gotosocial-admin-export).
+
+Advantages:
+
+* Database agnostic: exported data is in a somewhat generic format, and the `import` command can be used to insert this data into either a Postgres or an SQLite database.
+* Lightweight: only what is needed is preserved, so backup files can be quite small (even small enough to send in an email). Backup/import commands just take a few seconds to run.
+* Easily readable format: the output is just JSON.
+
+Disadvantages:
+
+* Loss of statuses/media/etc: don't do a backup/restore this way unless you're willing to drop stuff.
+* You need to use the GtS CLI tool to insert data back into a database, unless you write custom tooling for it.

docs/admin/cli.md

@@ -213,3 +213,78 @@ Example:
 ```bash
 gotosocial admin account password --username some_username --pasword some_really_good_password
 ```
+
+### gotosocial admin export
+
+This command can be used to export data from your GoToSocial instance into a file, for backup/storage.
+
+The file format will be a series of newline-separated JSON objects.
+
+`gotosocial admin export --help`:
+
+```text
+NAME:
+   gotosocial admin export - export data from the database to file at the given path
+
+USAGE:
+   gotosocial admin export [command options] [arguments...]
+
+OPTIONS:
+   --path value  the path of the file to import from/export to
+   --help, -h    show help (default: false)
+```
+
+Example:
+
+```bash
+gotosocial admin export --path ./example.json
+```
+
+`example.json`:
+
+```json
+{"type":"account","id":"01F8MH5NBDF2MV7CTC4Q5128HF","createdAt":"2021-08-31T12:00:53.985645Z","username":"1happyturtle","locked":true,"language":"en","uri":"http://localhost:8080/users/1happyturtle","url":"http://localhost:8080/@1happyturtle","inboxURI":"http://localhost:8080/users/1happyturtle/inbox","outboxURI":"http://localhost:8080/users/1happyturtle/outbox","followingUri":"http://localhost:8080/users/1happyturtle/following","followersUri":"http://localhost:8080/users/1happyturtle/followers","featuredCollectionUri":"http://localhost:8080/users/1happyturtle/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjz\nausfsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLz\neUPxdfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFx\njUz9l0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJY\nfKhKn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq\n79WbhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQABAoIBAGF+MxHjD15VV2NY\nKKb1GjMx98i1Xx6TijgoA+zmfha4LGu35e79Lql+0LXFp0zEpa6lAQsMQQhgd0OD\nmKKmSk+pxAvskJ4FxrhIf/yBFA4RMrj5OCaAOocRtdsOJ8n5UtFBrNAF0tzMY9q/\nkgzoq97aVF1mV9iFxaeBx6zT8ozSdqBq1PK/3w1dVg89S5tfKYc7Q0lQ00SfsTnd\niTDClKyqurebo9Pt6M7gXavgg3tvBlmwwr6XHs34Leng3oiN9mW8DVzaBMPzn+rE\nxF2eqs3v9vVpj8es88OwCh5P+ff8vJYvhu7Fcr/bJ8BItBQwfb8QBDATg/MXU2BI\n2ssW6AECgYEA4wmIyYGeu9+hzDa/J3Vh8GnlVNUCohHcChQdOsWsFXUgpVlUIHrX\neKHn42vD4Rzy52/YzJts4NkZTM9sL+kEXIEcpMG/S9xIIud7U0m/hMSAlmnJK/9j\niEXws3o4jo0E77jnRcBdIjpG4K5Eekm0DSR3SFhtZfEdN2DWPvu7K98CgYEA5tER\n/qJwFMc51AobMU87ZjXON7hI2U1WY/pVF62jSl0IcSsnj2riEKWLrs+GRG+HUg+U\naFSqAHcxaVHA0h0AYR8RopAhDdVKh0kvB8biLo+IEzNjPv2vyn0yRN5YSfXdGzyJ\nUjVU6kWdQOwmzy86nHgFaqEx7eofHIaGZzJK/AECgYEAu2VNQHX63TuzQuoVUa5z\nzoq5vhGsALYZF0CO98ndRkDNV22qIL0ESQ/qZS64GYFZhWouWoQXlGfdmCbFN65v\n6SKwz9UT3rvN1vGWO6Ltr9q6AG0EnYpJT1vbV2kUcaU4Y94NFue2d9/+TMnKv91B\n/m8Q/efvNGuWH/WQIaCKV6UCgYBz89WhYMMDfS4M2mLcu5vwddk53qciGxrqMMjs\nkzsz0Va7W12NS7lzeWaZlAE0gf6t98urOdUJVNeKvBoss4sMP0phqxwf0eWV3ur0\ncjIQB+TpGGikLVdRVuGY/UXHKe9AjoHBva8B3aTpB3lbnbNJBXZbIc1uYq3sa5w7\nXWWUAQKBgH3yW73RRpQNcc9hTUssomUsnQQgHxpfWx5tNxqod36Ytd9EKBh3NqUZ\nvPcH6gdh7mcnNaVNTtQOHLHsbPfBK/pqvb3MAsdlokJcQz8MQJ9SGBBPY6PaGw8z\nq/ambaQykER6dwlXTIlU20uXY0bttOL/iYjKmgo3vA66qfzS6nsg\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjzausf\nsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLzeUPx\ndfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFxjUz9\nl0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJYfKhK\nn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq79Wb\nhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/1happyturtle#main-key"}
+{"type":"account","id":"01F8MH0BBE4FHXPH513MBVFHB0","createdAt":"2021-09-08T10:00:53.985634Z","username":"weed_lord420","locked":true,"language":"en","uri":"http://localhost:8080/users/weed_lord420","url":"http://localhost:8080/@weed_lord420","inboxURI":"http://localhost:8080/users/weed_lord420/inbox","outboxURI":"http://localhost:8080/users/weed_lord420/outbox","followingUri":"http://localhost:8080/users/weed_lord420/following","followersUri":"http://localhost:8080/users/weed_lord420/followers","featuredCollectionUri":"http://localhost:8080/users/weed_lord420/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0b\nMIyLRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//P\nceYpo5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4\nus6VxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+\nfNyYVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPc\nqwtx0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQABAoIBAEAA4GHNS4k+Ke4j\nx4J0XkUjV5UbuPY0pSpSDjOJHOJmUfLcg85Ds9mYYO6zxwOaqmrC42ieclI5rh84\nTWQUqX9+VAk1J9UKeE4xZ1SSBtnZ3rK9PjrERZ+dmQ0dATaCuEO5Wwgu7Trk++Bg\nIqy8WNGZL94v9tfwALp1jTXW9AvmQoNdCFBP62vcmYW4YLjnggxLCFTA8YKfdePa\nTuxxY6uLkeBbxzWpbRU2+bmlxd5OnCkiRSMHIX+6JdtCu2JdWpUTCnWrFi2n1TZz\nZQx9z5rvowK1O785jGMFum5vBWpjIU8sJcXmPjGMU25zzmrhzfmkJsTXER3CXoUo\nSqSPqgECgYEA78OR7bY5KKQQ7Lyz6dru4Fct5P/OXTQoOg5aS7TKb95LVWj+TANn\n5djwIbLmAUV30z0Id9VgiZOL0Hny8+3VV9eU088Z408pAy5WQrL3dB8tZLUJSq5c\n5k6X15/VjWOOZKppDxShzoV3mcohrnwVwkv4fhPFQQOJJBYz6xurWs0CgYEA3MDE\nsDMd9ahzO0dl62ynojkkA8ZTcn2UdyvLpGj9UxT5j9vWF3CfqitXgcpNiVSIbxqQ\nbo/pBch7c/2Xakv5zkdcrJj5/6gyr+m1/tK2o7+CjDaSE4SYwufXx+qkl03Zpyzt\nKdOi7Hz/b2tdjump7ECEDE45mG2ea8oSnPgXl0cCgYBkGGFzu/9g2B24t47ksmHH\nhp3CXIjqoDurARLxSCi7SzJoFc0ULtfRPSAC8YzUOwwrQ++lF4+V3+MexcqHy2Kl\nqXqYcn18SC/3BAE/Fzf3Yoyw3mNiqihefbEmc7PTsxxfKkVx5ksmzNGBgsFM9sCe\nvNigyeAvpCo8xogmPwbqgQKBgE34mIBTzcUzFmBdu5YH7r3RyPK8XkUWLhZZlbgg\njTmHMw6o61mkIgENBf+F4RUckoQLsfAbTIcKZPB3JcAZzcYaVpVwAv1V/3E671lu\nO6xivE2iCL50GzDcis7GBhSbHsF5kNsxMV6uV9qW5ZjQ13/m2b0u9BDuxwHzgdeH\nmW2JAoGAIUOYniuEwdygxWVnYatpr3NPjT3BOKoV5i9zkeJRu1hFpwQM6vQ4Ds5p\nGC5vbMKAv9Cwuw62e2HvqTun3+U2Y5Uived3XCpgM/50BFrFHCfuqXEnu1bEzk5z\n9mIhp8uXPxzC5N7tRQfb3/eU1IUcb6T6ksbr2P81z0j03J55erg=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0bMIyL\nRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//PceYp\no5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4us6V\nxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+fNyY\nVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPcqwtx\n0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/weed_lord420#main-key"}
+{"type":"account","id":"01F8MH17FWEB39HZJ76B6VXSKF","createdAt":"2021-09-05T10:00:53.985641Z","username":"admin","locked":true,"language":"en","uri":"http://localhost:8080/users/admin","url":"http://localhost:8080/@admin","inboxURI":"http://localhost:8080/users/admin/inbox","outboxURI":"http://localhost:8080/users/admin/outbox","followingUri":"http://localhost:8080/users/admin/following","followersUri":"http://localhost:8080/users/admin/followers","featuredCollectionUri":"http://localhost:8080/users/admin/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVq\nhujDhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLR\nBI97qD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wg\nfvtEjEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G\n8kQJDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/Bk\nRhhGp2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQABAoIBAGK0aIADOU4ffJDe\n7sveiih5Fc1PATwx/QIR2QkWM1SREdx6LYclcX44V8xDanAbE44p1SkHY/CsEtYy\nXnyoXnn2FwFDQrdveY7+I6PApOPLAcKWkyLltC+hbVdj92/6YGNrm7EA/a77wruH\nmwjiivLnTG2CLecNiXSl33DA9YU4Yz+2Tza3IpTdjt8c/dz/BKKaxaWV+i9ew5VR\nioo5v51B+J8PrneCM/p8LGiLV148Njr0JqV6eFy1JuzItYMYdc3Fp+YnMzsuMZEA\n1akMcoln/ucVJyOFnCn6jx47nIoPZLl1KxX3aRDRfvrejm6W4yAkkTmR5voSRqax\njPL3rI0CgYEA9Acu4TO8xJ3uGaUad0N9JTYQVSmtAaE/g+df9LGMSzoj8X95S4xE\nQsGPqNGDm2VWADJjK4P05twZ+LfsfSKQ86wbp4/gbgnXpqB1P5Lty/B7KxiTnNwt\nwb1WGWTCukxfUSL3PRyf8uylkrg72RxKiBx4zKO3WVSLWOZWrFtn0qMCgYEA0H2p\nJs9Nv20ADOOX5tQ7+ruS6/B/Fhyj5fhflSYCAtOW7aME7+zQKJyqSQZ4b2Aub3Tp\nGIaUbRIGzjHyuTultFFWvjU3H5aI/0g1G9WKaBhNkyTIYVmMKtYyhXNvouWing8x\noraWx8TTBP8Cdnnk+QgdR2fpug8cghKupp5wvO8CgYA1JFtRL7MsHjh73TimQExA\njkWARlMmx7bNQtXis8eZmk+5h8kiaqly4DQoz3eZn7fa0x5Fm7b5j3UYdPVLSvvG\nFPTwyKRXUk1kPA1MivK+NuCbwf5jao+MYW8emJLPf1JCmRq+dD1g6aglC3n9Dewt\nOAYWipCjI4Y1FfRKFJ3HgQKBgEAb47+DTyzln3ZXJYZdDHR06SCTuwBZnixAy2NZ\nZJTp6yb3UbVU5E0Yn2QFEVNuB9lN4b8g4tMHEACnazN6G+HugPXL9z9HUqjs0yfT\n6dNIZdIxJUyJ9IfXhYFzlYhJhE+F7IVUD9kttJV8tI0pvja1QAuM8Fm9+84jYIDr\nh08RAoGAMYbjKHbtejcHBwt1kIcSss0cDmlZbBleJo8tdmdg4ndf5GE9N4/EL7tq\nm2zYSfr7OVdnOwRhoO+xF/6d1L7+TR1wz+k2fuMsI71aM5Ocp1nYTutjIkBTcldZ\nZzvjOgZWng5icuRLQQiDSKG5uqazqL/xGXkijb4kp4WW6myWY3c=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVqhujD\nhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLRBI97\nqD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wgfvtE\njEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G8kQJ\nDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/BkRhhG\np2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/admin#main-key"}
+{"type":"account","id":"01F8MH1H7YV1Z7D2C8K2730QBF","createdAt":"2021-09-06T10:00:53.985643Z","username":"the_mighty_zork","locked":true,"language":"en","uri":"http://localhost:8080/users/the_mighty_zork","url":"http://localhost:8080/@the_mighty_zork","inboxURI":"http://localhost:8080/users/the_mighty_zork/inbox","outboxURI":"http://localhost:8080/users/the_mighty_zork/outbox","followingUri":"http://localhost:8080/users/the_mighty_zork/following","followersUri":"http://localhost:8080/users/the_mighty_zork/followers","featuredCollectionUri":"http://localhost:8080/users/the_mighty_zork/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss\n5mEA/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvC\nC9zt/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZ\nFHptEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1\ntMhsUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlq\nefr58l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQABAoIBAFa+UypbFG1cW2Tr\nNBxPm7ngOEtXl8MicV4dIVKh0TwOo13ZxtNFBbOj7jALmPn/9HrtmbkABPQHDL1U\n/nt9aNSAeTjpwH3RaD5vFX3n0g8n2zJBOZLxxzAjNi4RBLYj5uP1AiKkdvRlsJza\nuSFDkty2zMBqN9mLPHE+RePj5Qa6tjYfIQqQzu/+YnYMlXHoC2yHNKsvz6S5FhVj\nv5zATv2JlJQH3RSmhuPOah73iQnKCLzYYEAHleawKrCg/rZ3ht37Guvabeq7MqQN\nvi9pJdAA+RMxPsboHajskePjOTYJgKQSxEAMRTMfBR40aZxklxQL0EoBd1Y3CHXh\nfMg0xWECgYEA0ORrpJ1A2WNQwKcDDeBBsaJqWF4EraoFzYrugKZrAYEeVyuGD0zq\nARUaWkZTZ1f6wQ10i1WxAuKlBEds7QsLdZzLsA4um4JlBroCZiYfPnmTtb8op1LY\nFqeYTByvAmnfWWTuOI67GX9ruLg8tEGuz38kuQVSxYs51its3tScNPUCgYEAyRst\nwRbqpOqnwoRoS6pxv0Vpc3nUcfaVYwsg/qobJkiwAdlUYeE7alvEY926VW4cvU/X\nhy3L1punAqnyLI7uuqCefXEbNxO0Cebyy4Kv2Ye1uzl0OHsJczSNdfpNqfAIKwtN\nHLCYDGCsluQhz+I/5Pd0dT+JDPPW9hKS2HG7o+kCgYBqugn1VRLo/sEnbS02TbnC\n1ESZWY/yWsgUOEObH2vUnO+vgeFAt/9nBi0sqnm6d0z6jbFZ7zI9UycUhJm2ksoM\nEUxQay6M7ZZIVYkcP6X++YbqePyAYOdey8oYOR+BkC45MkQ0SVh2so+LFTaOsnBq\nO3+7uGiN3ZBzSESbpO0acQKBgQCONrsXZeZO82XpB4tdns3LbgGRWKEkajTgEnml\nvZNvck2NMSwb/5PttbFe0ei4CyMluPV4MamJPQ9Qse+BFR67OWR63uZY/4T8z6X4\nxpUmZnLcUFfgrRlUr+AtgvEy8HxGPDquxC7x6deC6RcEFEIM3/UqCOEZGMJ1x1Ky\n31LLKQKBgGCKwVgQ8+4JyHZFkie3YdHhxJDokgY+Opb0HNnoBY/lZ54UMCCJQPS2\n0XPSu651j/3adr3RQneU04gF6U2/D5JzFEV0kUsqZ4Zy2EEU0LU4ibus0gyomSpK\niWhU4QrC/M4ELxYZinlNu3ThPWNQ/PMNteVWfdgOcV7uUWl0ViFp\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss5mEA\n/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvCC9zt\n/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZFHpt\nEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1tMhs\nUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlqefr5\n8l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/the_mighty_zork#main-key"}
+{"type":"block","id":"01FEXXET6XXMF7G2V3ASZP3YQW","createdAt":"2021-09-08T09:00:53.965362Z","uri":"http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW","accountId":"01F8MH5NBDF2MV7CTC4Q5128HF","targetAccountId":"01F8MH5ZK5VRH73AKHQM6Y9VNX"}
+{"type":"account","id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","createdAt":"2021-08-31T12:00:53.985646Z","username":"foss_satan","domain":"fossbros-anonymous.io","locked":true,"language":"en","uri":"http://fossbros-anonymous.io/users/foss_satan","url":"http://fossbros-anonymous.io/@foss_satan","inboxURI":"http://fossbros-anonymous.io/users/foss_satan/inbox","outboxURI":"http://fossbros-anonymous.io/users/foss_satan/outbox","followingUri":"http://fossbros-anonymous.io/users/foss_satan/following","followersUri":"http://fossbros-anonymous.io/users/foss_satan/followers","featuredCollectionUri":"http://fossbros-anonymous.io/users/foss_satan/collections/featured","actorType":"Person","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEA2OyVgkaIL9VohXKYTh319j4OouHRX/8QC7piXj71k7q5RDzEyvis\nVZBc5/C1/crCpxt895i0Ai2CiXQx+dISV7s/JBhAGl8s7TQ8jLlMuptrI0+sdkBC\nlu8pU0qQmoeXVnlquOzNmqGufUxIDtLXLZDN17qf/7vWA23q4d0tG5KQhGGGKiVM\n61Ufvr9MmgPBSpyUvYMAulFlz1264L49aGWeVgOz3qUQzqtxjrP0kaIbeyt56miP\nKr5AqkRgSsXci+FAo6suxR5gzo9NgleNkbZWF9MQyKlawukPwZUDSh396vtNQMee\n/4mto7mAXw8iio0IacrYO3F7iyewXnmI/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://fossbros-anonymous.io/users/foss_satan/main-key"}
+{"type":"follow","id":"01F8PYDCE8XE23GRE5DPZJDZDP","createdAt":"2021-09-08T09:00:54.749465Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PYDCE8XE23GRE5DPZJDZDP","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH5NBDF2MV7CTC4Q5128HF"}
+{"type":"follow","id":"01F8PY8RHWRQZV038T4E8T9YK8","createdAt":"2021-09-06T12:00:54.749459Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PY8RHWRQZV038T4E8T9YK8","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH17FWEB39HZJ76B6VXSKF"}
+{"type":"domainBlock","id":"01FF22EQM7X8E3RX1XGPN7S87D","createdAt":"2021-09-08T10:00:53.968971Z","domain":"replyguys.com","createdByAccountID":"01F8MH17FWEB39HZJ76B6VXSKF","privateComment":"i blocked this domain because they keep replying with pushy + unwarranted linux advice","publicComment":"reply-guying to tech posts","obfuscate":false}
+{"type":"user","id":"01F8MGYG9E893WRHW0TAEXR8GJ","createdAt":"2021-09-08T10:00:53.97247Z","accountID":"01F8MH0BBE4FHXPH513MBVFHB0","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","locale":"en","lastEmailedAt":"0001-01-01T00:00:00Z","confirmationToken":"a5a280bd-34be-44a3-8330-a57eaf61b8dd","confirmationTokenSentAt":"2021-09-08T10:00:53.972472Z","unconfirmedEmail":"weed_lord420@example.org","moderator":false,"admin":false,"disabled":false,"approved":false}
+{"type":"user","id":"01F8MGWYWKVKS3VS8DV1AMYPGE","createdAt":"2021-09-05T10:00:53.972475Z","email":"admin@example.org","accountID":"01F8MH17FWEB39HZJ76B6VXSKF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:50:53.972477Z","lastSignInAt":"2021-09-08T08:00:53.972477Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:30:53.972478Z","confirmedAt":"2021-09-05T10:00:53.972478Z","moderator":true,"admin":true,"disabled":false,"approved":true}
+{"type":"user","id":"01F8MGVGPHQ2D3P3X0454H54Z5","createdAt":"2021-09-06T22:00:53.97248Z","email":"zork@example.org","accountID":"01F8MH1H7YV1Z7D2C8K2730QBF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972481Z","lastSignInAt":"2021-09-08T08:00:53.972481Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972482Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972483Z","confirmedAt":"2021-09-07T00:00:53.972482Z","moderator":false,"admin":false,"disabled":false,"approved":true}
+{"type":"user","id":"01F8MH1VYJAE00TVVGMM5JNJ8X","createdAt":"2021-09-06T22:00:53.972485Z","email":"tortle.dude@example.org","accountID":"01F8MH5NBDF2MV7CTC4Q5128HF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972485Z","lastSignInAt":"2021-09-08T08:00:53.972486Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972487Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972487Z","confirmedAt":"2021-09-07T00:00:53.972487Z","moderator":false,"admin":false,"disabled":false,"approved":true}
+{"type":"instance","id":"01BZDDRPAB8J645ABY31HHF68Y","createdAt":"2021-09-08T10:00:54.763912Z","domain":"localhost:8080","title":"localhost:8080","uri":"http://localhost:8080","reputation":0}
+```
+
+### gotosocial admin import
+
+This command can be used to import data from a file into your GoToSocial database.
+
+If GoToSocial tables don't yet exist in the database, they will be created.
+
+If any conflicts occur while importing (an already exists while attempting to import a specific account, for example), then the process will be aborted.
+
+The file format should be a series of newline-separated JSON objects (see above).
+
+`gotosocial admin import --help`:
+
+```text
+NAME:
+   gotosocial admin import - import data from a file into the database
+
+USAGE:
+   gotosocial admin import [command options] [arguments...]
+
+OPTIONS:
+   --path value  the path of the file to import from/export to
+   --help, -h    show help (default: false)
+```
+
+Example:
+
+```bash
+gotosocial admin import --path ./example.json
+```

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.15
+	github.com/mitchellh/mapstructure v1.4.1
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646

go.sum

@@ -346,6 +346,8 @@ github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A
 github.com/memcachier/mc v2.0.1+incompatible/go.mod h1:7bkvFE61leUBvXz+yxsOnGBQSZpBSPIMUQSmmSHvuXc=
 github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY=
 github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

internal/api/s2s/user/repliesget_test.go

@@ -157,7 +157,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() {
 	b, err := ioutil.ReadAll(result.Body)
 	assert.NoError(suite.T(), err)
 
-	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
+	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
 
 	// should be a Collection
 	m := make(map[string]interface{})
@@ -188,7 +188,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
 	// setup request
 	recorder := httptest.NewRecorder()
 	ctx, _ := gin.CreateTestContext(recorder)
-	ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0", nil) // the endpoint we're hitting
 	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
 	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
 
@@ -220,7 +220,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
 	assert.NoError(suite.T(), err)
 
 	fmt.Println(string(b))
-	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
+	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
 
 	// should be a Collection
 	m := make(map[string]interface{})

internal/cache/status.go

@@ -144,7 +144,10 @@ func copyStatus(status *gtsmodel.Status) *gtsmodel.Status {
 		Sensitive:                status.Sensitive,
 		Language:                 status.Language,
 		CreatedWithApplicationID: status.CreatedWithApplicationID,
-		VisibilityAdvanced:       status.VisibilityAdvanced,
+		Federated:                status.Federated,
+		Boostable:                status.Boostable,
+		Replyable:                status.Replyable,
+		Likeable:                 status.Likeable,
 		ActivityStreamsType:      status.ActivityStreamsType,
 		Text:                     status.Text,
 		Pinned:                   status.Pinned,

internal/cliactions/admin/trans/export.go

@@ -0,0 +1,52 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/cliactions"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+)
+
+// Export exports info from the database into a file
+var Export cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
+	dbConn, err := bundb.NewBunDBService(ctx, c, log)
+	if err != nil {
+		return fmt.Errorf("error creating dbservice: %s", err)
+	}
+
+	exporter := trans.NewExporter(dbConn, log)
+
+	path, ok := c.ExportCLIFlags[config.TransPathFlag]
+	if !ok {
+		return errors.New("no path set")
+	}
+
+	if err := exporter.ExportMinimal(ctx, path); err != nil {
+		return err
+	}
+
+	return dbConn.Stop(ctx)
+}

internal/cliactions/admin/trans/import.go

@@ -0,0 +1,56 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/cliactions"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+)
+
+// Import imports info from a file into the database
+var Import cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
+	dbConn, err := bundb.NewBunDBService(ctx, c, log)
+	if err != nil {
+		return fmt.Errorf("error creating dbservice: %s", err)
+	}
+
+	importer := trans.NewImporter(dbConn, log)
+
+	path, ok := c.ExportCLIFlags[config.TransPathFlag]
+	if !ok {
+		return errors.New("no path set")
+	}
+
+	if err := dbConn.CreateAllTables(ctx); err != nil {
+		return err
+	}
+
+	if err := importer.Import(ctx, path); err != nil {
+		return err
+	}
+
+	return dbConn.Stop(ctx)
+}

internal/cliactions/server/server.go

@@ -39,7 +39,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/gotosocial"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/oidc"
@@ -51,32 +50,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/web"
 )
 
-var models []interface{} = []interface{}{
-	&gtsmodel.Account{},
-	&gtsmodel.Application{},
-	&gtsmodel.Block{},
-	&gtsmodel.DomainBlock{},
-	&gtsmodel.EmailDomainBlock{},
-	&gtsmodel.Follow{},
-	&gtsmodel.FollowRequest{},
-	&gtsmodel.MediaAttachment{},
-	&gtsmodel.Mention{},
-	&gtsmodel.Status{},
-	&gtsmodel.StatusToEmoji{},
-	&gtsmodel.StatusToTag{},
-	&gtsmodel.StatusFave{},
-	&gtsmodel.StatusBookmark{},
-	&gtsmodel.StatusMute{},
-	&gtsmodel.Tag{},
-	&gtsmodel.User{},
-	&gtsmodel.Emoji{},
-	&gtsmodel.Instance{},
-	&gtsmodel.Notification{},
-	&gtsmodel.RouterSession{},
-	&gtsmodel.Token{},
-	&gtsmodel.Client{},
-}
-
 // Start creates and starts a gotosocial server
 var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
 	dbService, err := bundb.NewBunDBService(ctx, c, log)
@@ -84,10 +57,8 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log
 		return fmt.Errorf("error creating dbservice: %s", err)
 	}
 
-	for _, m := range models {
+	if err := dbService.CreateAllTables(ctx); err != nil {
-		if err := dbService.CreateTable(ctx, m); err != nil {
+		return fmt.Errorf("error creating database tables: %s", err)
-			return fmt.Errorf("table creation error: %s", err)
-		}
 	}
 
 	if err := dbService.CreateInstanceAccount(ctx); err != nil {

internal/config/config.go

@@ -36,6 +36,9 @@ const (
 
 	PasswordFlag  = "password"
 	PasswordUsage = "the password to set for this account"
+
+	TransPathFlag  = "path"
+	TransPathUsage = "the path of the file to import from/export to"
 )
 
 // Config pulls together all the configuration needed to run gotosocial
@@ -65,6 +68,7 @@ type Config struct {
 		Not parsed from .yaml configuration file.
 	*/
 	AccountCLIFlags map[string]string
+	ExportCLIFlags  map[string]string
 	SoftwareVersion string
 }
 
@@ -92,6 +96,7 @@ func Empty() *Config {
 		LetsEncryptConfig: &LetsEncryptConfig{},
 		OIDCConfig:        &OIDCConfig{},
 		AccountCLIFlags:   make(map[string]string),
+		ExportCLIFlags:    make(map[string]string),
 	}
 }
 
@@ -320,6 +325,9 @@ func (c *Config) ParseCLIFlags(f KeyedFlags, version string) error {
 	c.AccountCLIFlags[EmailFlag] = f.String(EmailFlag)
 	c.AccountCLIFlags[PasswordFlag] = f.String(PasswordFlag)
 
+	// export CLI flags
+	c.ExportCLIFlags[TransPathFlag] = f.String(TransPathFlag)
+
 	c.SoftwareVersion = version
 	return nil
 }

internal/db/basic.go

@@ -26,6 +26,10 @@ type Basic interface {
 	// For implementations that don't use tables, this can just return nil.
 	CreateTable(ctx context.Context, i interface{}) Error
 
+	// CreateAllTables creates *all* tables necessary for the running of GoToSocial.
+	// Because it uses the 'if not exists' parameter it is safe to run against a GtS that's already been initialized.
+	CreateAllTables(ctx context.Context) Error
+
 	// DropTable drops the table for the given interface.
 	// For implementations that don't use tables, this can just return nil.
 	DropTable(ctx context.Context, i interface{}) Error

internal/db/bundb/basic.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/uptrace/bun"
 )
 
@@ -53,17 +54,8 @@ func (b *basicDB) GetWhere(ctx context.Context, where []db.Where, i interface{})
 	}
 
 	q := b.conn.NewSelect().Model(i)
-	for _, w := range where {
+
-		if w.Value == nil {
+	selectWhere(q, where)
-			q = q.Where("? IS NULL", bun.Ident(w.Key))
-		} else {
-			if w.CaseInsensitive {
-				q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value)
-			} else {
-				q = q.Where("? = ?", bun.Safe(w.Key), w.Value)
-			}
-		}
-	}
 
 	err := q.Scan(ctx)
 	return b.conn.ProcessError(err)
@@ -97,9 +89,7 @@ func (b *basicDB) DeleteWhere(ctx context.Context, where []db.Where, i interface
 		NewDelete().
 		Model(i)
 
-	for _, w := range where {
+	deleteWhere(q, where)
-		q = q.Where("? = ?", bun.Safe(w.Key), w.Value)
-	}
 
 	_, err := q.Exec(ctx)
 	return b.conn.ProcessError(err)
@@ -128,17 +118,7 @@ func (b *basicDB) UpdateOneByID(ctx context.Context, id string, key string, valu
 func (b *basicDB) UpdateWhere(ctx context.Context, where []db.Where, key string, value interface{}, i interface{}) db.Error {
 	q := b.conn.NewUpdate().Model(i)
 
-	for _, w := range where {
+	updateWhere(q, where)
-		if w.Value == nil {
-			q = q.Where("? IS NULL", bun.Ident(w.Key))
-		} else {
-			if w.CaseInsensitive {
-				q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value)
-			} else {
-				q = q.Where("? = ?", bun.Safe(w.Key), w.Value)
-			}
-		}
-	}
 
 	q = q.Set("? = ?", bun.Safe(key), value)
 
@@ -151,6 +131,40 @@ func (b *basicDB) CreateTable(ctx context.Context, i interface{}) db.Error {
 	return err
 }
 
+func (b *basicDB) CreateAllTables(ctx context.Context) db.Error {
+	models := []interface{}{
+		&gtsmodel.Account{},
+		&gtsmodel.Application{},
+		&gtsmodel.Block{},
+		&gtsmodel.DomainBlock{},
+		&gtsmodel.EmailDomainBlock{},
+		&gtsmodel.Follow{},
+		&gtsmodel.FollowRequest{},
+		&gtsmodel.MediaAttachment{},
+		&gtsmodel.Mention{},
+		&gtsmodel.Status{},
+		&gtsmodel.StatusToEmoji{},
+		&gtsmodel.StatusToTag{},
+		&gtsmodel.StatusFave{},
+		&gtsmodel.StatusBookmark{},
+		&gtsmodel.StatusMute{},
+		&gtsmodel.Tag{},
+		&gtsmodel.User{},
+		&gtsmodel.Emoji{},
+		&gtsmodel.Instance{},
+		&gtsmodel.Notification{},
+		&gtsmodel.RouterSession{},
+		&gtsmodel.Token{},
+		&gtsmodel.Client{},
+	}
+	for _, i := range models {
+		if err := b.CreateTable(ctx, i); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (b *basicDB) DropTable(ctx context.Context, i interface{}) db.Error {
 	_, err := b.conn.NewDropTable().Model(i).IfExists().Exec(ctx)
 	return b.conn.ProcessError(err)

internal/db/bundb/basic_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
@@ -42,7 +43,25 @@ func (suite *BasicTestSuite) TestGetAllStatuses() {
 	s := []*gtsmodel.Status{}
 	err := suite.db.GetAll(context.Background(), &s)
 	suite.NoError(err)
-	suite.Len(s, 12)
+	suite.Len(s, 13)
+}
+
+func (suite *BasicTestSuite) TestGetAllNotNull() {
+	where := []db.Where{{
+		Key:   "domain",
+		Value: nil,
+		Not:   true,
+	}}
+
+	a := []*gtsmodel.Account{}
+
+	err := suite.db.GetWhere(context.Background(), where, &a)
+	suite.NoError(err)
+	suite.NotEmpty(a)
+
+	for _, acct := range a {
+		suite.NotEmpty(acct.Domain)
+	}
 }
 
 func TestBasicTestSuite(t *testing.T) {

internal/db/bundb/status.go

@@ -240,11 +240,11 @@ func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status,
 			}
 		}
 
-		// only do one loop if we only want direct children
+		// if we're not only looking for direct children of status, then do the same children-finding
-		if onlyDirect {
+		// operation for the found child status too.
-			return
+		if !onlyDirect {
+			s.statusChildren(ctx, child, foundStatuses, false, minID)
 		}
-		s.statusChildren(ctx, child, foundStatuses, false, minID)
 	}
 }
 

internal/db/bundb/status_test.go

@@ -43,10 +43,14 @@ func (suite *StatusTestSuite) TestGetStatusByID() {
 	suite.Nil(status.BoostOfAccount)
 	suite.Nil(status.InReplyTo)
 	suite.Nil(status.InReplyToAccount)
+	suite.True(status.Federated)
+	suite.True(status.Boostable)
+	suite.True(status.Replyable)
+	suite.True(status.Likeable)
 }
 
 func (suite *StatusTestSuite) TestGetStatusByURI() {
-	status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_1_status_1"].URI)
+	status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_2_status_3"].URI)
 	if err != nil {
 		suite.FailNow(err.Error())
 	}
@@ -57,6 +61,10 @@ func (suite *StatusTestSuite) TestGetStatusByURI() {
 	suite.Nil(status.BoostOfAccount)
 	suite.Nil(status.InReplyTo)
 	suite.Nil(status.InReplyToAccount)
+	suite.True(status.Federated)
+	suite.True(status.Boostable)
+	suite.False(status.Replyable)
+	suite.False(status.Likeable)
 }
 
 func (suite *StatusTestSuite) TestGetStatusWithExtras() {
@@ -70,6 +78,10 @@ func (suite *StatusTestSuite) TestGetStatusWithExtras() {
 	suite.NotEmpty(status.Tags)
 	suite.NotEmpty(status.Attachments)
 	suite.NotEmpty(status.Emojis)
+	suite.True(status.Federated)
+	suite.True(status.Boostable)
+	suite.True(status.Replyable)
+	suite.True(status.Likeable)
 }
 
 func (suite *StatusTestSuite) TestGetStatusWithMention() {
@@ -83,6 +95,10 @@ func (suite *StatusTestSuite) TestGetStatusWithMention() {
 	suite.NotEmpty(status.MentionIDs)
 	suite.NotEmpty(status.InReplyToID)
 	suite.NotEmpty(status.InReplyToAccountID)
+	suite.True(status.Federated)
+	suite.True(status.Boostable)
+	suite.True(status.Replyable)
+	suite.True(status.Likeable)
 }
 
 func (suite *StatusTestSuite) TestGetStatusTwice() {
@@ -104,6 +120,18 @@ func (suite *StatusTestSuite) TestGetStatusTwice() {
 	suite.Less(duration2, duration1)
 }
 
+func (suite *StatusTestSuite) TestGetStatusChildren() {
+	targetStatus := suite.testStatuses["local_account_1_status_1"]
+	children, err := suite.db.GetStatusChildren(context.Background(), targetStatus, true, "")
+	suite.NoError(err)
+	suite.Len(children, 2)
+	for _, c := range children {
+		suite.Equal(targetStatus.URI, c.InReplyToURI)
+		suite.Equal(targetStatus.AccountID, c.InReplyToAccountID)
+		suite.Equal(targetStatus.ID, c.InReplyToID)
+	}
+}
+
 func TestStatusTestSuite(t *testing.T) {
 	suite.Run(t, new(StatusTestSuite))
 }

internal/db/bundb/util.go

@@ -19,6 +19,7 @@
 package bundb
 
 import (
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/uptrace/bun"
 )
 
@@ -35,3 +36,65 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery {
 			WhereOr("? = ''", bun.Ident(column))
 	}
 }
+
+// updateWhere parses []db.Where and adds it to the given update query.
+func updateWhere(q *bun.UpdateQuery, where []db.Where) {
+	for _, w := range where {
+		query, args := parseWhere(w)
+		q = q.Where(query, args...)
+	}
+}
+
+// selectWhere parses []db.Where and adds it to the given select query.
+func selectWhere(q *bun.SelectQuery, where []db.Where) {
+	for _, w := range where {
+		query, args := parseWhere(w)
+		q = q.Where(query, args...)
+	}
+}
+
+// deleteWhere parses []db.Where and adds it to the given where query.
+func deleteWhere(q *bun.DeleteQuery, where []db.Where) {
+	for _, w := range where {
+		query, args := parseWhere(w)
+		q = q.Where(query, args...)
+	}
+}
+
+// parseWhere looks through the options on a single db.Where entry, and
+// returns the appropriate query string and arguments.
+func parseWhere(w db.Where) (query string, args []interface{}) {
+	if w.Not {
+		if w.Value == nil {
+			query = "? IS NOT NULL"
+			args = []interface{}{bun.Ident(w.Key)}
+			return
+		}
+
+		if w.CaseInsensitive {
+			query = "LOWER(?) != LOWER(?)"
+			args = []interface{}{bun.Safe(w.Key), w.Value}
+			return
+		}
+
+		query = "? != ?"
+		args = []interface{}{bun.Safe(w.Key), w.Value}
+		return
+	}
+
+	if w.Value == nil {
+		query = "? IS NULL"
+		args = []interface{}{bun.Ident(w.Key)}
+		return
+	}
+
+	if w.CaseInsensitive {
+		query = "LOWER(?) = LOWER(?)"
+		args = []interface{}{bun.Safe(w.Key), w.Value}
+		return
+	}
+
+	query = "? = ?"
+	args = []interface{}{bun.Safe(w.Key), w.Value}
+	return
+}

internal/db/params.go

@@ -22,9 +22,13 @@ package db
 type Where struct {
 	// The table to search on.
 	Key string
-	// The value that must be set.
+	// The value to match.
 	Value interface{}
 	// Whether the value (if a string) should be case sensitive or not.
 	// Defaults to false.
 	CaseInsensitive bool
+	// If set, reverse the where.
+	// `WHERE k = v` becomes `WHERE k != v`.
+	// `WHERE k IS NULL` becomes `WHERE k IS NOT NULL`
+	Not bool
 }

internal/federation/dereferencing/announce.go

@@ -60,7 +60,10 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat
 	announce.BoostOfID = boostedStatus.ID
 	announce.BoostOfAccountID = boostedStatus.AccountID
 	announce.Visibility = boostedStatus.Visibility
-	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+	announce.Federated = boostedStatus.Federated
+	announce.Boostable = boostedStatus.Boostable
+	announce.Replyable = boostedStatus.Replyable
+	announce.Likeable = boostedStatus.Likeable
 	announce.BoostOf = boostedStatus
 	return nil
 }

internal/federation/dereferencing/status_test.go

@@ -58,6 +58,10 @@ func (suite *StatusTestSuite) TestDereferenceSimpleStatus() {
 	dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
 	suite.NoError(err)
 	suite.Equal(status.ID, dbStatus.ID)
+	suite.True(dbStatus.Federated)
+	suite.True(dbStatus.Boostable)
+	suite.True(dbStatus.Replyable)
+	suite.True(dbStatus.Likeable)
 
 	// account should be in the database now too
 	account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)
@@ -96,6 +100,10 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() {
 	dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
 	suite.NoError(err)
 	suite.Equal(status.ID, dbStatus.ID)
+	suite.True(dbStatus.Federated)
+	suite.True(dbStatus.Boostable)
+	suite.True(dbStatus.Replyable)
+	suite.True(dbStatus.Likeable)
 
 	// account should be in the database now too
 	account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)

internal/gtsmodel/status.go

@@ -57,10 +57,13 @@ type Status struct {
 	Language                 string             `validate:"-" bun:",nullzero"`                                                                         // what language is this status written in?
 	CreatedWithApplicationID string             `validate:"required_if=Local true,omitempty,ulid" bun:"type:CHAR(26),nullzero"`                        // Which application was used to create this status?
 	CreatedWithApplication   *Application       `validate:"-" bun:"rel:belongs-to"`                                                                    // application corresponding to createdWithApplicationID
-	VisibilityAdvanced       VisibilityAdvanced `validate:"required" bun:",nullzero,notnull" `                                                         // advanced visibility for this status
 	ActivityStreamsType      string             `validate:"required" bun:",nullzero,notnull"`                                                          // What is the activitystreams type of this status? See: https://www.w3.org/TR/activitystreams-vocabulary/#object-types. Will probably almost always be Note but who knows!.
 	Text                     string             `validate:"-" bun:",nullzero"`                                                                         // Original text of the status without formatting
-	Pinned                   bool               `validate:"-" bun:",notnull,default:false" `                                                           // Has this status been pinned by its owner?
+	Pinned                   bool               `validate:"-" bun:",notnull,default:false"`                                                            // Has this status been pinned by its owner?
+	Federated                bool               `validate:"-" bun:",notnull"`                                                                          // This status will be federated beyond the local timeline(s)
+	Boostable                bool               `validate:"-" bun:",notnull"`                                                                          // This status can be boosted/reblogged
+	Replyable                bool               `validate:"-" bun:",notnull"`                                                                          // This status can be replied to
+	Likeable                 bool               `validate:"-" bun:",notnull"`                                                                          // This status can be liked/faved
 }
 
 // StatusToTag is an intermediate struct to facilitate the many2many relationship between a status and one or more tags.
@@ -96,21 +99,3 @@ const (
 	// VisibilityDefault is used when no other setting can be found.
 	VisibilityDefault Visibility = VisibilityUnlocked
 )
-
-// VisibilityAdvanced models flags for fine-tuning visibility and interactivity of a status.
-//
-// All flags default to true.
-//
-// If PUBLIC is selected, flags will all be overwritten to TRUE regardless of what is selected.
-//
-// If UNLOCKED is selected, any flags can be turned on or off in any combination.
-//
-// If FOLLOWERS-ONLY or MUTUALS-ONLY are selected, boostable will always be FALSE. Other flags can be turned on or off as desired.
-//
-// If DIRECT is selected, boostable will be FALSE, and all other flags will be TRUE.
-type VisibilityAdvanced struct {
-	Federated bool `validate:"-" bun:",notnull,default:true"` // This status will be federated beyond the local timeline(s)
-	Boostable bool `validate:"-" bun:",notnull,default:true"` // This status can be boosted/reblogged
-	Replyable bool `validate:"-" bun:",notnull,default:true"` // This status can be replied to
-	Likeable  bool `validate:"-" bun:",notnull,default:true"` // This status can be liked/faved
-}

internal/processing/fromclientapi.go

@@ -51,7 +51,7 @@ func (p *processor) ProcessFromClientAPI(ctx context.Context, clientMsg messages
 				return err
 			}
 
-			if status.VisibilityAdvanced.Federated {
+			if status.Federated {
 				return p.federateStatus(ctx, status)
 			}
 		case ap.ActivityFollow:

internal/processing/status/boost.go

@@ -46,7 +46,7 @@ func (p *processor) Boost(ctx context.Context, requestingAccount *gtsmodel.Accou
 	if !visible {
 		return nil, gtserror.NewErrorNotFound(errors.New("status is not visible"))
 	}
-	if !targetStatus.VisibilityAdvanced.Boostable {
+	if !targetStatus.Boostable {
 		return nil, gtserror.NewErrorForbidden(errors.New("status is not boostable"))
 	}
 

internal/processing/status/fave.go

@@ -49,7 +49,7 @@ func (p *processor) Fave(ctx context.Context, requestingAccount *gtsmodel.Accoun
 	if !visible {
 		return nil, gtserror.NewErrorNotFound(errors.New("status is not visible"))
 	}
-	if !targetStatus.VisibilityAdvanced.Likeable {
+	if !targetStatus.Likeable {
 		return nil, gtserror.NewErrorForbidden(errors.New("status is not faveable"))
 	}
 

internal/processing/status/util.go

@@ -33,12 +33,10 @@ import (
 
 func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
 	// by default all flags are set to true
-	gtsAdvancedVis := gtsmodel.VisibilityAdvanced{
+	federated := true
-		Federated: true,
+	boostable := true
-		Boostable: true,
+	replyable := true
-		Replyable: true,
+	likeable := true
-		Likeable:  true,
-	}
 
 	var vis gtsmodel.Visibility
 	// If visibility isn't set on the form, then just take the account default.
@@ -58,47 +56,50 @@ func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.Advanc
 	case gtsmodel.VisibilityUnlocked:
 		// for unlocked the user can set any combination of flags they like so look at them all to see if they're set and then apply them
 		if form.Federated != nil {
-			gtsAdvancedVis.Federated = *form.Federated
+			federated = *form.Federated
 		}
 
 		if form.Boostable != nil {
-			gtsAdvancedVis.Boostable = *form.Boostable
+			boostable = *form.Boostable
 		}
 
 		if form.Replyable != nil {
-			gtsAdvancedVis.Replyable = *form.Replyable
+			replyable = *form.Replyable
 		}
 
 		if form.Likeable != nil {
-			gtsAdvancedVis.Likeable = *form.Likeable
+			likeable = *form.Likeable
 		}
 
 	case gtsmodel.VisibilityFollowersOnly, gtsmodel.VisibilityMutualsOnly:
 		// for followers or mutuals only, boostable will *always* be false, but the other fields can be set so check and apply them
-		gtsAdvancedVis.Boostable = false
+		boostable = false
 
 		if form.Federated != nil {
-			gtsAdvancedVis.Federated = *form.Federated
+			federated = *form.Federated
 		}
 
 		if form.Replyable != nil {
-			gtsAdvancedVis.Replyable = *form.Replyable
+			replyable = *form.Replyable
 		}
 
 		if form.Likeable != nil {
-			gtsAdvancedVis.Likeable = *form.Likeable
+			likeable = *form.Likeable
 		}
 
 	case gtsmodel.VisibilityDirect:
 		// direct is pretty easy: there's only one possible setting so return it
-		gtsAdvancedVis.Federated = true
+		federated = true
-		gtsAdvancedVis.Boostable = false
+		boostable = false
-		gtsAdvancedVis.Federated = true
+		replyable = true
-		gtsAdvancedVis.Likeable = true
+		likeable = true
 	}
 
 	status.Visibility = vis
-	status.VisibilityAdvanced = gtsAdvancedVis
+	status.Federated = federated
+	status.Boostable = boostable
+	status.Replyable = replyable
+	status.Likeable = likeable
 	return nil
 }
 
@@ -123,7 +124,7 @@ func (p *processor) ProcessReplyToID(ctx context.Context, form *apimodel.Advance
 		}
 		return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err)
 	}
-	if !repliedStatus.VisibilityAdvanced.Replyable {
+	if !repliedStatus.Replyable {
 		return fmt.Errorf("status with id %s is marked as not replyable", form.InReplyToID)
 	}
 

internal/timeline/get_test.go

@@ -73,8 +73,8 @@ func (suite *GetTestSuite) TestGetDefault() {
 		suite.FailNow(err.Error())
 	}
 
-	// we only have 12 statuses in the test suite
+	// we only have 13 statuses in the test suite
-	suite.Len(statuses, 12)
+	suite.Len(statuses, 13)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string
@@ -166,8 +166,8 @@ func (suite *GetTestSuite) TestGetMinID() {
 		suite.FailNow(err.Error())
 	}
 
-	// we should only get 5 statuses back, since we asked for a min ID that excludes some of our entries
+	// we should only get 6 statuses back, since we asked for a min ID that excludes some of our entries
-	suite.Len(statuses, 5)
+	suite.Len(statuses, 6)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string
@@ -188,8 +188,8 @@ func (suite *GetTestSuite) TestGetSinceID() {
 		suite.FailNow(err.Error())
 	}
 
-	// we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries
+	// we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries
-	suite.Len(statuses, 5)
+	suite.Len(statuses, 6)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string
@@ -210,8 +210,8 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() {
 		suite.FailNow(err.Error())
 	}
 
-	// we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries
+	// we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries
-	suite.Len(statuses, 5)
+	suite.Len(statuses, 6)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string

internal/timeline/index_test.go

@@ -66,7 +66,7 @@ func (suite *IndexTestSuite) TestIndexBeforeLowID() {
 	// the oldest indexed post should be the lowest one we have in our testrig
 	postID, err := suite.timeline.OldestIndexedPostID(context.Background())
 	suite.NoError(err)
-	suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", postID)
+	suite.Equal("01F8MHAMCHF6Y650WCRSCP4WMY", postID)
 
 	indexLength := suite.timeline.PostIndexLength(context.Background())
 	suite.Equal(10, indexLength)
@@ -95,7 +95,7 @@ func (suite *IndexTestSuite) TestIndexBehindHighID() {
 	// the newest indexed post should be the highest one we have in our testrig
 	postID, err := suite.timeline.NewestIndexedPostID(context.Background())
 	suite.NoError(err)
-	suite.Equal("01FCTA44PW9H1TB328S9AQXKDS", postID)
+	suite.Equal("01FF25D5Q0DH7CHD57CTRS6WK0", postID)
 
 	// indexLength should be 10 because that's all this user has hometimelineable
 	indexLength := suite.timeline.PostIndexLength(context.Background())

internal/timeline/manager_test.go

@@ -67,9 +67,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 	err = suite.manager.PrepareXFromTop(context.Background(), testAccount.ID, 20)
 	suite.NoError(err)
 
-	// local_account_1 can see 12 statuses out of the testrig statuses in its home timeline
+	// local_account_1 can see 13 statuses out of the testrig statuses in its home timeline
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(12, indexedLen)
+	suite.Equal(13, indexedLen)
 
 	// oldest should now be set
 	oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
@@ -79,7 +79,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 	// get hometimeline
 	statuses, err := suite.manager.HomeTimeline(context.Background(), testAccount.ID, "", "", "", 20, false)
 	suite.NoError(err)
-	suite.Len(statuses, 12)
+	suite.Len(statuses, 13)
 
 	// now wipe the last status from all timelines, as though it had been deleted by the owner
 	err = suite.manager.WipeStatusFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R")
@@ -87,7 +87,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 
 	// timeline should be shorter
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(11, indexedLen)
+	suite.Equal(12, indexedLen)
 
 	// oldest should now be different
 	oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
@@ -101,7 +101,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 
 	// timeline should be shorter
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(10, indexedLen)
+	suite.Equal(11, indexedLen)
 
 	// oldest should now be different
 	oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
@@ -112,9 +112,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 	err = suite.manager.WipeStatusesFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID)
 	suite.NoError(err)
 
-	// timeline should be empty now
+	// timeline should be shorter
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(5, indexedLen)
+	suite.Equal(6, indexedLen)
 
 	// ingest 1 into the timeline
 	status1 := suite.testStatuses["admin_account_status_1"]
@@ -130,7 +130,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 
 	// timeline should be longer now
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(7, indexedLen)
+	suite.Equal(8, indexedLen)
 
 	// try to ingest status 2 again
 	ingested, err = suite.manager.IngestAndPrepare(context.Background(), status2, testAccount.ID)

internal/trans/decoders.go

@@ -0,0 +1,138 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mitchellh/mapstructure"
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func newDecoder(target interface{}) (*mapstructure.Decoder, error) {
+	decoderConfig := &mapstructure.DecoderConfig{
+		DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), // this is needed to decode time.Time entries serialized as string
+		Result:     target,
+	}
+	return mapstructure.NewDecoder(decoderConfig)
+}
+
+func (i *importer) accountDecode(e transmodel.Entry) (*transmodel.Account, error) {
+	a := &transmodel.Account{}
+	if err := i.simpleDecode(e, a); err != nil {
+		return nil, err
+	}
+
+	// extract public key
+	publicKeyBlock, _ := pem.Decode([]byte(a.PublicKeyString))
+	if publicKeyBlock == nil {
+		return nil, errors.New("accountDecode: error decoding account public key")
+	}
+	publicKey, err := x509.ParsePKCS1PublicKey(publicKeyBlock.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("accountDecode: error parsing account public key: %s", err)
+	}
+	a.PublicKey = publicKey
+
+	if a.Domain == "" {
+		// extract private key (local account)
+		privateKeyBlock, _ := pem.Decode([]byte(a.PrivateKeyString))
+		if privateKeyBlock == nil {
+			return nil, errors.New("accountDecode: error decoding account private key")
+		}
+		privateKey, err := x509.ParsePKCS1PrivateKey(privateKeyBlock.Bytes)
+		if err != nil {
+			return nil, fmt.Errorf("accountDecode: error parsing account private key: %s", err)
+		}
+		a.PrivateKey = privateKey
+	}
+
+	return a, nil
+}
+
+func (i *importer) blockDecode(e transmodel.Entry) (*transmodel.Block, error) {
+	b := &transmodel.Block{}
+	if err := i.simpleDecode(e, b); err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (i *importer) domainBlockDecode(e transmodel.Entry) (*transmodel.DomainBlock, error) {
+	b := &transmodel.DomainBlock{}
+	if err := i.simpleDecode(e, b); err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (i *importer) followDecode(e transmodel.Entry) (*transmodel.Follow, error) {
+	f := &transmodel.Follow{}
+	if err := i.simpleDecode(e, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func (i *importer) followRequestDecode(e transmodel.Entry) (*transmodel.FollowRequest, error) {
+	f := &transmodel.FollowRequest{}
+	if err := i.simpleDecode(e, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func (i *importer) instanceDecode(e transmodel.Entry) (*transmodel.Instance, error) {
+	inst := &transmodel.Instance{}
+	if err := i.simpleDecode(e, inst); err != nil {
+		return nil, err
+	}
+
+	return inst, nil
+}
+
+func (i *importer) userDecode(e transmodel.Entry) (*transmodel.User, error) {
+	u := &transmodel.User{}
+	if err := i.simpleDecode(e, u); err != nil {
+		return nil, err
+	}
+
+	return u, nil
+}
+
+func (i *importer) simpleDecode(entry transmodel.Entry, target interface{}) error {
+	decoder, err := newDecoder(target)
+	if err != nil {
+		return fmt.Errorf("simpleDecode: error creating decoder: %s", err)
+	}
+
+	if err := decoder.Decode(&entry); err != nil {
+		return fmt.Errorf("simpleDecode: error decoding: %s", err)
+	}
+
+	return nil
+}

internal/trans/encoders.go

@@ -0,0 +1,83 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"os"
+
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+// accountEncode handles special fields like private + public keys on accounts
+func (e *exporter) accountEncode(ctx context.Context, f *os.File, a *transmodel.Account) error {
+	a.Type = transmodel.TransAccount
+
+	// marshal public key
+	encodedPublicKey := x509.MarshalPKCS1PublicKey(a.PublicKey)
+	if encodedPublicKey == nil {
+		return errors.New("could not MarshalPKCS1PublicKey")
+	}
+	publicKeyBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PUBLIC KEY",
+		Bytes: encodedPublicKey,
+	})
+	a.PublicKeyString = string(publicKeyBytes)
+
+	if a.Domain == "" {
+		// marshal private key for local account
+		encodedPrivateKey := x509.MarshalPKCS1PrivateKey(a.PrivateKey)
+		if encodedPrivateKey == nil {
+			return errors.New("could not MarshalPKCS1PrivateKey")
+		}
+		privateKeyBytes := pem.EncodeToMemory(&pem.Block{
+			Type:  "RSA PRIVATE KEY",
+			Bytes: encodedPrivateKey,
+		})
+		a.PrivateKeyString = string(privateKeyBytes)
+	}
+
+	return e.simpleEncode(ctx, f, a, a.ID)
+}
+
+// simpleEncode can be used for any type that doesn't have special keys which need handling differently,
+// or for types where special keys have already been handled.
+//
+// Beware, the 'type' key on the passed interface should already have been set, since simpleEncode won't know
+// what type it is! If you try to decode stuff you've encoded with a missing type key, you're going to have a bad time.
+func (e *exporter) simpleEncode(ctx context.Context, file *os.File, i interface{}, id string) error {
+	_, alreadyWritten := e.writtenIDs[id]
+	if alreadyWritten {
+		// this exporter has already exported an entry with this ID, no need to do it twice
+		return nil
+	}
+
+	err := json.NewEncoder(file).Encode(i)
+	if err != nil {
+		return fmt.Errorf("simpleEncode: error encoding entry with id %s: %s", id, err)
+	}
+
+	e.writtenIDs[id] = true
+	return nil
+}

internal/trans/export.go

@@ -0,0 +1,223 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/package trans
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, file *os.File) ([]*transmodel.Account, error) {
+	// select using the 'where' we've been provided
+	accounts := []*transmodel.Account{}
+	if err := e.db.GetWhere(ctx, where, &accounts); err != nil {
+		return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err)
+	}
+
+	// write any accounts found to file
+	for _, a := range accounts {
+		if err := e.accountEncode(ctx, file, a); err != nil {
+			return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err)
+		}
+	}
+
+	return accounts, nil
+}
+
+func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Block, error) {
+	blocksUnique := make(map[string]*transmodel.Block)
+
+	// for each account we want to export both where it's blocking and where it's blocked
+	for _, a := range accounts {
+		// 1. export blocks owned by given account
+		whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}}
+		blocking := []*transmodel.Block{}
+		if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err)
+		}
+		for _, b := range blocking {
+			b.Type = transmodel.TransBlock
+			if err := e.simpleEncode(ctx, file, b, b.ID); err != nil {
+				return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err)
+			}
+			blocksUnique[b.ID] = b
+		}
+
+		// 2. export blocks that target given account
+		whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		blocked := []*transmodel.Block{}
+		if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err)
+		}
+		for _, b := range blocked {
+			b.Type = transmodel.TransBlock
+			if err := e.simpleEncode(ctx, file, b, b.ID); err != nil {
+				return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err)
+			}
+			blocksUnique[b.ID] = b
+		}
+	}
+
+	// now return all the blocks we found
+	blocks := []*transmodel.Block{}
+	for _, b := range blocksUnique {
+		blocks = append(blocks, b)
+	}
+
+	return blocks, nil
+}
+
+func (e *exporter) exportDomainBlocks(ctx context.Context, file *os.File) ([]*transmodel.DomainBlock, error) {
+	domainBlocks := []*transmodel.DomainBlock{}
+
+	if err := e.db.GetAll(ctx, &domainBlocks); err != nil {
+		return nil, fmt.Errorf("exportBlocks: error selecting domain blocks: %s", err)
+	}
+
+	for _, b := range domainBlocks {
+		b.Type = transmodel.TransDomainBlock
+		if err := e.simpleEncode(ctx, file, b, b.ID); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error encoding domain block: %s", err)
+		}
+	}
+
+	return domainBlocks, nil
+}
+
+func (e *exporter) exportFollows(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Follow, error) {
+	followsUnique := make(map[string]*transmodel.Follow)
+
+	// for each account we want to export both where it's following and where it's followed
+	for _, a := range accounts {
+		// 1. export follows owned by given account
+		whereFollowing := []db.Where{{Key: "account_id", Value: a.ID}}
+		following := []*transmodel.Follow{}
+		if err := e.db.GetWhere(ctx, whereFollowing, &following); err != nil {
+			return nil, fmt.Errorf("exportFollows: error selecting follows owned by account %s: %s", a.ID, err)
+		}
+		for _, follow := range following {
+			follow.Type = transmodel.TransFollow
+			if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil {
+				return nil, fmt.Errorf("exportFollows: error encoding follow owned by account %s: %s", a.ID, err)
+			}
+			followsUnique[follow.ID] = follow
+		}
+
+		// 2. export follows that target given account
+		whereFollowed := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		followed := []*transmodel.Follow{}
+		if err := e.db.GetWhere(ctx, whereFollowed, &followed); err != nil {
+			return nil, fmt.Errorf("exportFollows: error selecting follows targeting account %s: %s", a.ID, err)
+		}
+		for _, follow := range followed {
+			follow.Type = transmodel.TransFollow
+			if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil {
+				return nil, fmt.Errorf("exportFollows: error encoding follow targeting account %s: %s", a.ID, err)
+			}
+			followsUnique[follow.ID] = follow
+		}
+	}
+
+	// now return all the follows we found
+	follows := []*transmodel.Follow{}
+	for _, follow := range followsUnique {
+		follows = append(follows, follow)
+	}
+
+	return follows, nil
+}
+
+func (e *exporter) exportFollowRequests(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.FollowRequest, error) {
+	frsUnique := make(map[string]*transmodel.FollowRequest)
+
+	// for each account we want to export both where it's following and where it's followed
+	for _, a := range accounts {
+		// 1. export follow requests owned by given account
+		whereRequesting := []db.Where{{Key: "account_id", Value: a.ID}}
+		requesting := []*transmodel.FollowRequest{}
+		if err := e.db.GetWhere(ctx, whereRequesting, &requesting); err != nil {
+			return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests owned by account %s: %s", a.ID, err)
+		}
+		for _, fr := range requesting {
+			fr.Type = transmodel.TransFollowRequest
+			if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil {
+				return nil, fmt.Errorf("exportFollowRequests: error encoding follow request owned by account %s: %s", a.ID, err)
+			}
+			frsUnique[fr.ID] = fr
+		}
+
+		// 2. export follow requests that target given account
+		whereRequested := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		requested := []*transmodel.FollowRequest{}
+		if err := e.db.GetWhere(ctx, whereRequested, &requested); err != nil {
+			return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests targeting account %s: %s", a.ID, err)
+		}
+		for _, fr := range requested {
+			fr.Type = transmodel.TransFollowRequest
+			if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil {
+				return nil, fmt.Errorf("exportFollowRequests: error encoding follow request targeting account %s: %s", a.ID, err)
+			}
+			frsUnique[fr.ID] = fr
+		}
+	}
+
+	// now return all the followRequests we found
+	followRequests := []*transmodel.FollowRequest{}
+	for _, fr := range frsUnique {
+		followRequests = append(followRequests, fr)
+	}
+
+	return followRequests, nil
+}
+
+func (e *exporter) exportInstances(ctx context.Context, file *os.File) ([]*transmodel.Instance, error) {
+	instances := []*transmodel.Instance{}
+
+	if err := e.db.GetAll(ctx, &instances); err != nil {
+		return nil, fmt.Errorf("exportInstances: error selecting instance: %s", err)
+	}
+
+	for _, u := range instances {
+		u.Type = transmodel.TransInstance
+		if err := e.simpleEncode(ctx, file, u, u.ID); err != nil {
+			return nil, fmt.Errorf("exportInstances: error encoding instance: %s", err)
+		}
+	}
+
+	return instances, nil
+}
+
+func (e *exporter) exportUsers(ctx context.Context, file *os.File) ([]*transmodel.User, error) {
+	users := []*transmodel.User{}
+
+	if err := e.db.GetAll(ctx, &users); err != nil {
+		return nil, fmt.Errorf("exportUsers: error selecting users: %s", err)
+	}
+
+	for _, u := range users {
+		u.Type = transmodel.TransUser
+		if err := e.simpleEncode(ctx, file, u, u.ID); err != nil {
+			return nil, fmt.Errorf("exportUsers: error encoding user: %s", err)
+		}
+	}
+
+	return users, nil
+}

internal/trans/exporter.go

@@ -0,0 +1,46 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+// Exporter wraps functionality for exporting entries from the database to a file.
+type Exporter interface {
+	ExportMinimal(ctx context.Context, path string) error
+}
+
+type exporter struct {
+	db         db.DB
+	log        *logrus.Logger
+	writtenIDs map[string]bool
+}
+
+// NewExporter returns a new Exporter that will use the given db and logger.
+func NewExporter(db db.DB, log *logrus.Logger) Exporter {
+	return &exporter{
+		db:         db,
+		log:        log,
+		writtenIDs: make(map[string]bool),
+	}
+}

internal/trans/exportminimal.go

@@ -0,0 +1,150 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+func (e *exporter) ExportMinimal(ctx context.Context, path string) error {
+	if path == "" {
+		return errors.New("ExportMinimal: path empty")
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: couldn't export to %s: %s", path, err)
+	}
+
+	// export all local accounts we have in the database
+	localAccounts, err := e.exportAccounts(ctx, []db.Where{{Key: "domain", Value: nil}}, file)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting accounts: %s", err)
+	}
+
+	// export all blocks that relate to local accounts
+	blocks, err := e.exportBlocks(ctx, localAccounts, file)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting blocks: %s", err)
+	}
+
+	// for each block, make sure we've written out the account owning it, or targeted by it --
+	// this might include non-local accounts, but we need these so we don't lose anything
+	for _, b := range blocks {
+		_, alreadyWritten := e.writtenIDs[b.AccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.AccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting block owner account: %s", err)
+			}
+		}
+
+		_, alreadyWritten = e.writtenIDs[b.TargetAccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.TargetAccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting block target account: %s", err)
+			}
+		}
+	}
+
+	// export all follows that relate to local accounts
+	follows, err := e.exportFollows(ctx, localAccounts, file)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting follows: %s", err)
+	}
+
+	// for each follow, make sure we've written out the account owning it, or targeted by it --
+	// this might include non-local accounts, but we need these so we don't lose anything
+	for _, follow := range follows {
+		_, alreadyWritten := e.writtenIDs[follow.AccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.AccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow owner account: %s", err)
+			}
+		}
+
+		_, alreadyWritten = e.writtenIDs[follow.TargetAccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.TargetAccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow target account: %s", err)
+			}
+		}
+	}
+
+	// export all follow requests that relate to local accounts
+	followRequests, err := e.exportFollowRequests(ctx, localAccounts, file)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting follow requests: %s", err)
+	}
+
+	// for each follow request, make sure we've written out the account owning it, or targeted by it --
+	// this might include non-local accounts, but we need these so we don't lose anything
+	for _, fr := range followRequests {
+		_, alreadyWritten := e.writtenIDs[fr.AccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.AccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow request owner account: %s", err)
+			}
+		}
+
+		_, alreadyWritten = e.writtenIDs[fr.TargetAccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.TargetAccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow request target account: %s", err)
+			}
+		}
+	}
+
+	// export all domain blocks
+	if _, err := e.exportDomainBlocks(ctx, file); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting domain blocks: %s", err)
+	}
+
+	// export all users
+	if _, err := e.exportUsers(ctx, file); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting users: %s", err)
+	}
+
+	// export all instances
+	if _, err := e.exportInstances(ctx, file); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting instances: %s", err)
+	}
+
+	// export all SUSPENDED accounts to make sure the suspension sticks across db migration etc
+	whereSuspended := []db.Where{{
+		Key:   "suspended_at",
+		Not:   true,
+		Value: nil,
+	}}
+	if _, err := e.exportAccounts(ctx, whereSuspended, file); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting suspended accounts: %s", err)
+	}
+
+	return neatClose(file)
+}

internal/trans/exportminimal_test.go

@@ -0,0 +1,54 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+)
+
+type ExportMinimalTestSuite struct {
+	TransTestSuite
+}
+
+func (suite *ExportMinimalTestSuite) TestExportMinimalOK() {
+	// use a temporary file path that will be cleaned when the test is closed
+	tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString())
+
+	// export to the tempFilePath
+	exporter := trans.NewExporter(suite.db, suite.log)
+	err := exporter.ExportMinimal(context.Background(), tempFilePath)
+	suite.NoError(err)
+
+	// we should have some bytes in that file now
+	b, err := os.ReadFile(tempFilePath)
+	suite.NoError(err)
+	suite.NotEmpty(b)
+	fmt.Println(string(b))
+}
+
+func TestExportMinimalTestSuite(t *testing.T) {
+	suite.Run(t, &ExportMinimalTestSuite{})
+}

internal/trans/import.go

@@ -0,0 +1,146 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func (i *importer) Import(ctx context.Context, path string) error {
+	if path == "" {
+		return errors.New("Export: path empty")
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("Import: couldn't export to %s: %s", path, err)
+	}
+
+	decoder := json.NewDecoder(file)
+	decoder.UseNumber()
+
+	for {
+		entry := transmodel.Entry{}
+		err := decoder.Decode(&entry)
+		if err != nil {
+			if err == io.EOF {
+				i.log.Infof("Import: reached end of file")
+				return neatClose(file)
+			}
+			return fmt.Errorf("Import: error decoding in readLoop: %s", err)
+		}
+		if err := i.inputEntry(ctx, entry); err != nil {
+			return fmt.Errorf("Import: error inputting entry: %s", err)
+		}
+	}
+}
+
+func (i *importer) inputEntry(ctx context.Context, entry transmodel.Entry) error {
+	t, ok := entry[transmodel.TypeKey].(string)
+	if !ok {
+		return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json")
+	}
+
+	switch transmodel.Type(t) {
+	case transmodel.TransAccount:
+		account, err := i.accountDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into account: %s", err)
+		}
+		if err := i.putInDB(ctx, account); err != nil {
+			return fmt.Errorf("inputEntry: error adding account to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added account with id %s", account.ID)
+		return nil
+	case transmodel.TransBlock:
+		block, err := i.blockDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into block: %s", err)
+		}
+		if err := i.putInDB(ctx, block); err != nil {
+			return fmt.Errorf("inputEntry: error adding block to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added block with id %s", block.ID)
+		return nil
+	case transmodel.TransDomainBlock:
+		block, err := i.domainBlockDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into domain block: %s", err)
+		}
+		if err := i.putInDB(ctx, block); err != nil {
+			return fmt.Errorf("inputEntry: error adding domain block to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added domain block with id %s", block.ID)
+		return nil
+	case transmodel.TransFollow:
+		follow, err := i.followDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into follow: %s", err)
+		}
+		if err := i.putInDB(ctx, follow); err != nil {
+			return fmt.Errorf("inputEntry: error adding follow to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added follow with id %s", follow.ID)
+		return nil
+	case transmodel.TransFollowRequest:
+		fr, err := i.followRequestDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into follow request: %s", err)
+		}
+		if err := i.putInDB(ctx, fr); err != nil {
+			return fmt.Errorf("inputEntry: error adding follow request to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added follow request with id %s", fr.ID)
+		return nil
+	case transmodel.TransInstance:
+		inst, err := i.instanceDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into instance: %s", err)
+		}
+		if err := i.putInDB(ctx, inst); err != nil {
+			return fmt.Errorf("inputEntry: error adding instance to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added instance with id %s", inst.ID)
+		return nil
+	case transmodel.TransUser:
+		user, err := i.userDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into user: %s", err)
+		}
+		if err := i.putInDB(ctx, user); err != nil {
+			return fmt.Errorf("inputEntry: error adding user to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added user with id %s", user.ID)
+		return nil
+	}
+
+	i.log.Errorf("inputEntry: didn't recognize transtype '%s', skipping it", t)
+	return nil
+}
+
+func (i *importer) putInDB(ctx context.Context, entry interface{}) error {
+	return i.db.Put(ctx, entry)
+}

internal/trans/import_test.go

@@ -0,0 +1,91 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type ImportMinimalTestSuite struct {
+	TransTestSuite
+}
+
+func (suite *ImportMinimalTestSuite) TestImportMinimalOK() {
+	ctx := context.Background()
+
+	// use a temporary file path
+	tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString())
+
+	// export to the tempFilePath
+	exporter := trans.NewExporter(suite.db, suite.log)
+	err := exporter.ExportMinimal(ctx, tempFilePath)
+	suite.NoError(err)
+
+	// we should have some bytes in that file now
+	b, err := os.ReadFile(tempFilePath)
+	suite.NoError(err)
+	suite.NotEmpty(b)
+	fmt.Println(string(b))
+
+	// create a new database with just the tables created, no entries
+	testrig.StandardDBTeardown(suite.db)
+	newDB := testrig.NewTestDB()
+	testrig.CreateTestTables(newDB)
+
+	importer := trans.NewImporter(newDB, suite.log)
+	err = importer.Import(ctx, tempFilePath)
+	suite.NoError(err)
+
+	// we should have some accounts in the database
+	accounts := []*gtsmodel.Account{}
+	err = newDB.GetAll(ctx, &accounts)
+	suite.NoError(err)
+	suite.NotEmpty(accounts)
+
+	// we should have some blocks in the database
+	blocks := []*gtsmodel.Block{}
+	err = newDB.GetAll(ctx, &blocks)
+	suite.NoError(err)
+	suite.NotEmpty(blocks)
+
+	// we should have some follows in the database
+	follows := []*gtsmodel.Follow{}
+	err = newDB.GetAll(ctx, &follows)
+	suite.NoError(err)
+	suite.NotEmpty(follows)
+
+	// we should have some domain blocks in the database
+	domainBlocks := []*gtsmodel.DomainBlock{}
+	err = newDB.GetAll(ctx, &domainBlocks)
+	suite.NoError(err)
+	suite.NotEmpty(domainBlocks)
+}
+
+func TestImportMinimalTestSuite(t *testing.T) {
+	suite.Run(t, &ImportMinimalTestSuite{})
+}

internal/trans/importer.go

@@ -0,0 +1,44 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+// Importer wraps functionality for importing entries from a file into the database.
+type Importer interface {
+	Import(ctx context.Context, path string) error
+}
+
+type importer struct {
+	db  db.DB
+	log *logrus.Logger
+}
+
+// NewImporter returns a new Importer interface that uses the given db and logger.
+func NewImporter(db db.DB, log *logrus.Logger) Importer {
+	return &importer{
+		db:  db,
+		log: log,
+	}
+}

internal/trans/model/account.go

@@ -0,0 +1,54 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"crypto/rsa"
+	"time"
+)
+
+// Account represents the minimum viable representation of an account for export/import.
+type Account struct {
+	Type                  Type            `json:"type" bun:"-"`
+	ID                    string          `json:"id" bun:",nullzero"`
+	CreatedAt             *time.Time      `json:"createdAt" bun:",nullzero"`
+	Username              string          `json:"username" bun:",nullzero"`
+	DisplayName           string          `json:"displayName,omitempty" bun:",nullzero"`
+	Note                  string          `json:"note,omitempty" bun:",nullzero"`
+	Domain                string          `json:"domain,omitempty" bun:",nullzero"`
+	HeaderRemoteURL       string          `json:"headerRemoteURL,omitempty" bun:",nullzero"`
+	AvatarRemoteURL       string          `json:"avatarRemoteURL,omitempty" bun:",nullzero"`
+	Locked                bool            `json:"locked"`
+	Language              string          `json:"language,omitempty" bun:",nullzero"`
+	URI                   string          `json:"uri" bun:",nullzero"`
+	URL                   string          `json:"url" bun:",nullzero"`
+	InboxURI              string          `json:"inboxURI" bun:",nullzero"`
+	OutboxURI             string          `json:"outboxURI" bun:",nullzero"`
+	FollowingURI          string          `json:"followingUri" bun:",nullzero"`
+	FollowersURI          string          `json:"followersUri" bun:",nullzero"`
+	FeaturedCollectionURI string          `json:"featuredCollectionUri" bun:",nullzero"`
+	ActorType             string          `json:"actorType" bun:",nullzero"`
+	PrivateKey            *rsa.PrivateKey `json:"-" mapstructure:"-"`
+	PrivateKeyString      string          `json:"privateKey,omitempty" mapstructure:"privateKey" bun:"-"`
+	PublicKey             *rsa.PublicKey  `json:"-" mapstructure:"-"`
+	PublicKeyString       string          `json:"publicKey,omitempty" mapstructure:"publicKey" bun:"-"`
+	PublicKeyURI          string          `json:"publicKeyUri" bun:",nullzero"`
+	SuspendedAt           *time.Time      `json:"suspendedAt,omitempty" bun:",nullzero"`
+	SuspensionOrigin      string          `json:"suspensionOrigin,omitempty" bun:",nullzero"`
+}

internal/trans/model/block.go

@@ -0,0 +1,31 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+// Block represents an account block as serialized in an exported file.
+type Block struct {
+	Type            Type       `json:"type" bun:"-"`
+	ID              string     `json:"id" bun:",nullzero"`
+	CreatedAt       *time.Time `json:"createdAt" bun:",nullzero"`
+	URI             string     `json:"uri" bun:",nullzero"`
+	AccountID       string     `json:"accountId" bun:",nullzero"`
+	TargetAccountID string     `json:"targetAccountId" bun:",nullzero"`
+}

internal/trans/model/domainblock.go

@@ -0,0 +1,34 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+// DomainBlock represents a domain block as serialized in an exported file.
+type DomainBlock struct {
+	Type               Type       `json:"type" bun:"-"`
+	ID                 string     `json:"id" bun:",nullzero"`
+	CreatedAt          *time.Time `json:"createdAt" bun:",nullzero"`
+	Domain             string     `json:"domain" bun:",nullzero"`
+	CreatedByAccountID string     `json:"createdByAccountID" bun:",nullzero"`
+	PrivateComment     string     `json:"privateComment,omitempty" bun:",nullzero"`
+	PublicComment      string     `json:"publicComment,omitempty" bun:",nullzero"`
+	Obfuscate          bool       `json:"obfuscate" bun:",nullzero"`
+	SubscriptionID     string     `json:"subscriptionID,omitempty" bun:",nullzero"`
+}

internal/trans/model/follow.go

@@ -0,0 +1,31 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+// Follow represents an account follow as serialized in an export file.
+type Follow struct {
+	Type            Type       `json:"type" bun:"-"`
+	ID              string     `json:"id" bun:",nullzero"`
+	CreatedAt       *time.Time `json:"createdAt" bun:",nullzero"`
+	URI             string     `json:"uri" bun:",nullzero"`
+	AccountID       string     `json:"accountId" bun:",nullzero"`
+	TargetAccountID string     `json:"targetAccountId" bun:",nullzero"`
+}

internal/trans/model/followrequest.go

@@ -0,0 +1,31 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+// FollowRequest represents an account follow request as serialized in an export file.
+type FollowRequest struct {
+	Type            Type       `json:"type" bun:"-"`
+	ID              string     `json:"id" bun:",nullzero"`
+	CreatedAt       *time.Time `json:"createdAt" bun:",nullzero"`
+	URI             string     `json:"uri" bun:",nullzero"`
+	AccountID       string     `json:"accountId" bun:",nullzero"`
+	TargetAccountID string     `json:"targetAccountId" bun:",nullzero"`
+}

internal/trans/model/instance.go

@@ -0,0 +1,43 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"time"
+)
+
+// Instance represents an instance entry as serialized in an export file.
+type Instance struct {
+	Type                   Type       `json:"type" bun:"-"`
+	ID                     string     `json:"id" bun:",nullzero"`
+	CreatedAt              *time.Time `json:"createdAt" bun:",nullzero"`
+	Domain                 string     `json:"domain" bun:",nullzero"`
+	Title                  string     `json:"title,omitempty" bun:",nullzero"`
+	URI                    string     `json:"uri" bun:",nullzero"`
+	SuspendedAt            *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"`
+	DomainBlockID          string     `json:"domainBlockID,omitempty" bun:",nullzero"`
+	ShortDescription       string     `json:"shortDescription,omitempty" bun:",nullzero"`
+	Description            string     `json:"description,omitempty" bun:",nullzero"`
+	Terms                  string     `json:"terms,omitempty" bun:",nullzero"`
+	ContactEmail           string     `json:"contactEmail,omitempty" bun:",nullzero"`
+	ContactAccountUsername string     `json:"contactAccountUsername,omitempty" bun:",nullzero"`
+	ContactAccountID       string     `json:"contactAccountID,omitempty" bun:",nullzero"`
+	Reputation             int64      `json:"reputation"`
+	Version                string     `json:"version,omitempty" bun:",nullzero"`
+}

internal/trans/model/type.go

@@ -0,0 +1,41 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+// TypeKey should be set on a TransEntry to indicate the type of entry it is.
+const TypeKey = "type"
+
+// Type describes the type of a trans entry, and how it should be read/serialized.
+type Type string
+
+// Type of the trans entry. Describes how it should be read from file.
+const (
+	TransAccount          Type = "account"
+	TransBlock            Type = "block"
+	TransDomainBlock      Type = "domainBlock"
+	TransEmailDomainBlock Type = "emailDomainBlock"
+	TransFollow           Type = "follow"
+	TransFollowRequest    Type = "followRequest"
+	TransInstance         Type = "instance"
+	TransUser             Type = "user"
+)
+
+// Entry is used for deserializing trans entries into a rough interface so that
+// the TypeKey can be fetched, before continuing with full parsing.
+type Entry map[string]interface{}

internal/trans/model/user.go

@@ -0,0 +1,50 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"time"
+)
+
+// User represents a local instance user as serialized to an export file.
+type User struct {
+	Type                Type       `json:"type" bun:"-"`
+	ID                  string     `json:"id" bun:",nullzero"`
+	CreatedAt           *time.Time `json:"createdAt" bun:",nullzero"`
+	Email               string     `json:"email,omitempty" bun:",nullzero"`
+	AccountID           string     `json:"accountID" bun:",nullzero"`
+	EncryptedPassword   string     `json:"encryptedPassword" bun:",nullzero"`
+	CurrentSignInAt     *time.Time `json:"currentSignInAt,omitempty" bun:",nullzero"`
+	LastSignInAt        *time.Time `json:"lastSignInAt,omitempty" bun:",nullzero"`
+	InviteID            string     `json:"inviteID,omitempty" bun:",nullzero"`
+	ChosenLanguages     []string   `json:"chosenLanguages,omitempty" bun:",nullzero"`
+	FilteredLanguages   []string   `json:"filteredLanguage,omitempty" bun:",nullzero"`
+	Locale              string     `json:"locale" bun:",nullzero"`
+	LastEmailedAt       time.Time  `json:"lastEmailedAt,omitempty" bun:",nullzero"`
+	ConfirmationToken   string     `json:"confirmationToken,omitempty" bun:",nullzero"`
+	ConfirmationSentAt  *time.Time `json:"confirmationTokenSentAt,omitempty" bun:",nullzero"`
+	ConfirmedAt         *time.Time `json:"confirmedAt,omitempty" bun:",nullzero"`
+	UnconfirmedEmail    string     `json:"unconfirmedEmail,omitempty" bun:",nullzero"`
+	Moderator           bool       `json:"moderator"`
+	Admin               bool       `json:"admin"`
+	Disabled            bool       `json:"disabled"`
+	Approved            bool       `json:"approved"`
+	ResetPasswordToken  string     `json:"resetPasswordToken,omitempty" bun:",nullzero"`
+	ResetPasswordSentAt *time.Time `json:"resetPasswordSentAt,omitempty" bun:",nullzero"`
+}

internal/trans/trans_test.go

@@ -0,0 +1,42 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans_test
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type TransTestSuite struct {
+	suite.Suite
+	db  db.DB
+	log *logrus.Logger
+}
+
+func (suite *TransTestSuite) SetupTest() {
+	suite.db = testrig.NewTestDB()
+	suite.log = testrig.NewTestLog()
+	testrig.StandardDBSetup(suite.db, nil)
+}
+
+func (suite *TransTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+}

internal/trans/util.go

@@ -0,0 +1,32 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"fmt"
+	"os"
+)
+
+func neatClose(f *os.File) error {
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("error closing file: %s", err)
+	}
+
+	return nil
+}

internal/typeutils/astointernal.go

@@ -325,6 +325,11 @@ func (c *converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab
 
 	// advanced visibility for this status
 	// TODO: a lot of work to be done here -- a new type needs to be created for this in go-fed/activity using ASTOOL
+	// for now we just set everything to true
+	status.Federated = true
+	status.Boostable = true
+	status.Replyable = true
+	status.Likeable = true
 
 	// sensitive
 	// TODO: this is a bool

internal/typeutils/internal.go

@@ -70,7 +70,10 @@ func (c *converter) StatusToBoost(ctx context.Context, s *gtsmodel.Status, boost
 		BoostOfID:           s.ID,
 		BoostOfAccountID:    s.AccountID,
 		Visibility:          s.Visibility,
-		VisibilityAdvanced:  s.VisibilityAdvanced,
+		Federated:           s.Federated,
+		Boostable:           s.Boostable,
+		Replyable:           s.Replyable,
+		Likeable:            s.Likeable,
 
 		// attach these here for convenience -- the boosted status/account won't go in the DB
 		// but they're needed in the processor and for the frontend. Since we have them, we can

internal/validate/status_test.go

@@ -63,15 +63,13 @@ func happyStatus() *gtsmodel.Status {
 		Language:                 "en",
 		CreatedWithApplicationID: "01FEBBZHF4GFVRXSJVXD0JTZZ2",
 		CreatedWithApplication:   nil,
-		VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+		Federated:                true,
-			Federated: true,
+		Boostable:                true,
-			Boostable: true,
+		Replyable:                true,
-			Replyable: true,
+		Likeable:                 true,
-			Likeable:  true,
+		ActivityStreamsType:      ap.ObjectNote,
-		},
+		Text:                     "Test status! #hello",
-		ActivityStreamsType: ap.ObjectNote,
+		Pinned:                   false,
-		Text:                "Test status! #hello",
-		Pinned:              false,
 	}
 }
 

testrig/db.go

@@ -72,6 +72,16 @@ func NewTestDB() db.DB {
 	return testDB
 }
 
+// CreateTestTables creates prerequisite test tables in the database, but doesn't populate them.
+func CreateTestTables(db db.DB) {
+	ctx := context.Background()
+	for _, m := range testModels {
+		if err := db.CreateTable(ctx, m); err != nil {
+			logrus.Panicf("error creating table for %+v: %s", m, err)
+		}
+	}
+}
+
 // StandardDBSetup populates a given db with all the necessary tables/models for perfoming tests.
 //
 // The accounts parameter is provided in case the db should be populated with a certain set of accounts.
@@ -85,13 +95,9 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
 		logrus.Panic("db setup: db was nil")
 	}
 
-	ctx := context.Background()
+	CreateTestTables(db)
 
-	for _, m := range testModels {
+	ctx := context.Background()
-		if err := db.CreateTable(ctx, m); err != nil {
-			logrus.Panicf("error creating table for %+v: %s", m, err)
-		}
-	}
 
 	for _, v := range NewTestTokens() {
 		if err := db.Put(ctx, v); err != nil {
@@ -111,6 +117,18 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
 		}
 	}
 
+	for _, v := range NewTestBlocks() {
+		if err := db.Put(ctx, v); err != nil {
+			logrus.Panic(err)
+		}
+	}
+
+	for _, v := range NewTestDomainBlocks() {
+		if err := db.Put(ctx, v); err != nil {
+			logrus.Panic(err)
+		}
+	}
+
 	for _, v := range NewTestUsers() {
 		if err := db.Put(ctx, v); err != nil {
 			logrus.Panic(err)

testrig/testmodels.go

@@ -736,6 +736,19 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji {
 	}
 }
 
+func NewTestDomainBlocks() map[string]*gtsmodel.DomainBlock {
+	return map[string]*gtsmodel.DomainBlock{
+		"replyguys.com": {
+			ID:                 "01FF22EQM7X8E3RX1XGPN7S87D",
+			Domain:             "replyguys.com",
+			CreatedByAccountID: "01F8MH17FWEB39HZJ76B6VXSKF",
+			PrivateComment:     "i blocked this domain because they keep replying with pushy + unwarranted linux advice",
+			PublicComment:      "reply-guying to tech posts",
+			Obfuscate:          false,
+		},
+	}
+}
+
 type filenames struct {
 	Original string
 	Small    string
@@ -803,13 +816,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                true,
-				Replyable: true,
+			Likeable:                 true,
-				Likeable:  true,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 		"admin_account_status_2": {
 			ID:                       "01F8MHAAY43M6RJ473VQFCVH37",
@@ -828,13 +839,36 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                true,
-				Replyable: true,
+			Likeable:                 true,
-				Likeable:  true,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
+		},
-			ActivityStreamsType: ap.ObjectNote,
+		"admin_account_status_3": {
+			ID:                       "01FF25D5Q0DH7CHD57CTRS6WK0",
+			URI:                      "http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0",
+			URL:                      "http://localhost:8080/@admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0",
+			Content:                  "hi @the_mighty_zork welcome to the instance!",
+			CreatedAt:                time.Now().Add(-46 * time.Hour),
+			UpdatedAt:                time.Now().Add(-46 * time.Hour),
+			Local:                    true,
+			AccountURI:               "http://localhost:8080/users/admin",
+			MentionIDs:               []string{"01FF26A6BGEKCZFWNEHXB2ZZ6M"},
+			AccountID:                "01F8MH17FWEB39HZJ76B6VXSKF",
+			InReplyToID:              "01F8MHAMCHF6Y650WCRSCP4WMY",
+			InReplyToAccountID:       "01F8MH1H7YV1Z7D2C8K2730QBF",
+			InReplyToURI:             "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
+			BoostOfID:                "",
+			Visibility:               gtsmodel.VisibilityPublic,
+			Sensitive:                false,
+			Language:                 "en",
+			CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F",
+			Federated:                true,
+			Boostable:                true,
+			Replyable:                true,
+			Likeable:                 true,
+			ActivityStreamsType:      ap.ObjectNote,
 		},
 		"local_account_1_status_1": {
 			ID:                       "01F8MHAMCHF6Y650WCRSCP4WMY",
@@ -853,13 +887,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                true,
-				Replyable: true,
+			Likeable:                 true,
-				Likeable:  true,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_1_status_2": {
 			ID:                       "01F8MHAYFKS4KMXF8K5Y1C0KRN",
@@ -878,13 +910,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                false,
-				Federated: false,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                true,
-				Replyable: true,
+			Likeable:                 true,
-				Likeable:  true,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_1_status_3": {
 			ID:                       "01F8MHBBN8120SYH7D5S050MGK",
@@ -903,13 +933,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                false,
-				Boostable: false,
+			Replyable:                false,
-				Replyable: false,
+			Likeable:                 false,
-				Likeable:  false,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_1_status_4": {
 			ID:                       "01F8MH82FYRXD2RC6108DAJ5HB",
@@ -929,13 +957,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                true,
-				Replyable: true,
+			Likeable:                 true,
-				Likeable:  true,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_1_status_5": {
 			ID:                       "01FCTA44PW9H1TB328S9AQXKDS",
@@ -955,13 +981,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                true,
-				Replyable: true,
+			Likeable:                 true,
-				Likeable:  true,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_2_status_1": {
 			ID:                       "01F8MHBQCBTDKN6X5VHGMMN4MA",
@@ -980,13 +1004,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                true,
-				Replyable: true,
+			Likeable:                 true,
-				Likeable:  true,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_2_status_2": {
 			ID:                       "01F8MHC0H0A7XHTVH5F596ZKBM",
@@ -1005,13 +1027,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                false,
-				Replyable: false,
+			Likeable:                 true,
-				Likeable:  true,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_2_status_3": {
 			ID:                       "01F8MHC8VWDRBQR0N1BATDDEM5",
@@ -1030,13 +1050,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                false,
-				Replyable: false,
+			Likeable:                 false,
-				Likeable:  false,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_2_status_4": {
 			ID:                       "01F8MHCP5P2NWYQ416SBA0XSEV",
@@ -1055,12 +1073,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                true,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                false,
-				Federated: false,
+			Boostable:                false,
-				Boostable: false,
+			Replyable:                true,
-				Replyable: true,
+			Likeable:                 true,
-				Likeable:  true,
+
-			},
 			ActivityStreamsType: ap.ObjectNote,
 		},
 		"local_account_2_status_5": {
@@ -1083,13 +1100,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			Sensitive:                false,
 			Language:                 "en",
 			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
-			VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
+			Federated:                true,
-				Federated: true,
+			Boostable:                true,
-				Boostable: true,
+			Replyable:                true,
-				Replyable: true,
+			Likeable:                 true,
-				Likeable:  true,
+			ActivityStreamsType:      ap.ObjectNote,
-			},
-			ActivityStreamsType: ap.ObjectNote,
 		},
 	}
 }
@@ -1149,6 +1164,18 @@ func NewTestMentions() map[string]*gtsmodel.Mention {
 			TargetAccountURI: "http://localhost:8080/users/the_mighty_zork",
 			TargetAccountURL: "http://localhost:8080/@the_mighty_zork",
 		},
+		"admin_account_mention_zork": {
+			ID:               "01FF26A6BGEKCZFWNEHXB2ZZ6M",
+			StatusID:         "01FF25D5Q0DH7CHD57CTRS6WK0",
+			CreatedAt:        time.Now().Add(-46 * time.Hour),
+			UpdatedAt:        time.Now().Add(-46 * time.Hour),
+			OriginAccountID:  "01F8MH17FWEB39HZJ76B6VXSKF",
+			OriginAccountURI: "http://localhost:8080/users/admin",
+			TargetAccountID:  "01F8MH1H7YV1Z7D2C8K2730QBF",
+			NameString:       "@the_mighty_zork",
+			TargetAccountURI: "http://localhost:8080/users/the_mighty_zork",
+			TargetAccountURL: "http://localhost:8080/@the_mighty_zork",
+		},
 	}
 }
 
@@ -1215,6 +1242,19 @@ func NewTestFollows() map[string]*gtsmodel.Follow {
 	}
 }
 
+func NewTestBlocks() map[string]*gtsmodel.Block {
+	return map[string]*gtsmodel.Block{
+		"local_account_2_block_remote_account_1": {
+			ID:              "01FEXXET6XXMF7G2V3ASZP3YQW",
+			CreatedAt:       time.Now().Add(-1 * time.Hour),
+			UpdatedAt:       time.Now().Add(-1 * time.Hour),
+			URI:             "http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW",
+			AccountID:       "01F8MH5NBDF2MV7CTC4Q5128HF",
+			TargetAccountID: "01F8MH5ZK5VRH73AKHQM6Y9VNX",
+		},
+	}
+}
+
 // ActivityWithSignature wraps a pub.Activity along with its signature headers, for testing.
 type ActivityWithSignature struct {
 	Activity        pub.Activity
@@ -1374,7 +1414,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
 		DateHeader:      date,
 	}
 
-	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5")
+	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0")
 	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
 	fossSatanDereferenceLocalAccount1Status1RepliesLast := ActivityWithSignature{
 		SignatureHeader: sig,

vendor/github.com/mitchellh/mapstructure/CHANGELOG.md

@@ -0,0 +1,73 @@
+## unreleased
+
+* Fix regression where `*time.Time` value would be set to empty and not be sent 
+  to decode hooks properly [GH-232]
+
+## 1.4.0
+
+* A new decode hook type `DecodeHookFuncValue` has been added that has
+  access to the full values. [GH-183]
+* Squash is now supported with embedded fields that are struct pointers [GH-205]
+* Empty strings will convert to 0 for all numeric types when weakly decoding [GH-206]
+
+## 1.3.3
+
+* Decoding maps from maps creates a settable value for decode hooks [GH-203]
+
+## 1.3.2
+
+* Decode into interface type with a struct value is supported [GH-187]
+
+## 1.3.1
+
+* Squash should only squash embedded structs. [GH-194]
+
+## 1.3.0
+
+* Added `",omitempty"` support. This will ignore zero values in the source
+  structure when encoding. [GH-145]
+
+## 1.2.3
+
+* Fix duplicate entries in Keys list with pointer values. [GH-185]
+
+## 1.2.2
+
+* Do not add unsettable (unexported) values to the unused metadata key
+  or "remain" value. [GH-150]
+
+## 1.2.1
+
+* Go modules checksum mismatch fix
+
+## 1.2.0
+
+* Added support to capture unused values in a field using the `",remain"` value
+  in the mapstructure tag. There is an example to showcase usage.
+* Added `DecoderConfig` option to always squash embedded structs
+* `json.Number` can decode into `uint` types
+* Empty slices are preserved and not replaced with nil slices
+* Fix panic that can occur in when decoding a map into a nil slice of structs
+* Improved package documentation for godoc
+
+## 1.1.2
+
+* Fix error when decode hook decodes interface implementation into interface
+  type. [GH-140]
+
+## 1.1.1
+
+* Fix panic that can happen in `decodePtr`
+
+## 1.1.0
+
+* Added `StringToIPHookFunc` to convert `string` to `net.IP` and `net.IPNet` [GH-133]
+* Support struct to struct decoding [GH-137]
+* If source map value is nil, then destination map value is nil (instead of empty)
+* If source slice value is nil, then destination slice value is nil (instead of empty)
+* If source pointer is nil, then destination pointer is set to nil (instead of
+  allocated zero value of type)
+
+## 1.0.0
+
+* Initial tagged stable release.

vendor/github.com/mitchellh/mapstructure/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Mitchell Hashimoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/mitchellh/mapstructure/README.md

@@ -0,0 +1,46 @@
+# mapstructure [![Godoc](https://godoc.org/github.com/mitchellh/mapstructure?status.svg)](https://godoc.org/github.com/mitchellh/mapstructure)
+
+mapstructure is a Go library for decoding generic map values to structures
+and vice versa, while providing helpful error handling.
+
+This library is most useful when decoding values from some data stream (JSON,
+Gob, etc.) where you don't _quite_ know the structure of the underlying data
+until you read a part of it. You can therefore read a `map[string]interface{}`
+and use this library to decode it into the proper underlying native Go
+structure.
+
+## Installation
+
+Standard `go get`:
+
+```
+$ go get github.com/mitchellh/mapstructure
+```
+
+## Usage & Example
+
+For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/mapstructure).
+
+The `Decode` function has examples associated with it there.
+
+## But Why?!
+
+Go offers fantastic standard libraries for decoding formats such as JSON.
+The standard method is to have a struct pre-created, and populate that struct
+from the bytes of the encoded format. This is great, but the problem is if
+you have configuration or an encoding that changes slightly depending on
+specific fields. For example, consider this JSON:
+
+```json
+{
+  "type": "person",
+  "name": "Mitchell"
+}
+```
+
+Perhaps we can't populate a specific structure without first reading
+the "type" field from the JSON. We could always do two passes over the
+decoding of the JSON (reading the "type" first, and the rest later).
+However, it is much simpler to just decode this into a `map[string]interface{}`
+structure, read the "type" key, then use something like this library
+to decode it into the proper structure.

vendor/github.com/mitchellh/mapstructure/decode_hooks.go

@@ -0,0 +1,256 @@
+package mapstructure
+
+import (
+	"encoding"
+	"errors"
+	"fmt"
+	"net"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// typedDecodeHook takes a raw DecodeHookFunc (an interface{}) and turns
+// it into the proper DecodeHookFunc type, such as DecodeHookFuncType.
+func typedDecodeHook(h DecodeHookFunc) DecodeHookFunc {
+	// Create variables here so we can reference them with the reflect pkg
+	var f1 DecodeHookFuncType
+	var f2 DecodeHookFuncKind
+	var f3 DecodeHookFuncValue
+
+	// Fill in the variables into this interface and the rest is done
+	// automatically using the reflect package.
+	potential := []interface{}{f1, f2, f3}
+
+	v := reflect.ValueOf(h)
+	vt := v.Type()
+	for _, raw := range potential {
+		pt := reflect.ValueOf(raw).Type()
+		if vt.ConvertibleTo(pt) {
+			return v.Convert(pt).Interface()
+		}
+	}
+
+	return nil
+}
+
+// DecodeHookExec executes the given decode hook. This should be used
+// since it'll naturally degrade to the older backwards compatible DecodeHookFunc
+// that took reflect.Kind instead of reflect.Type.
+func DecodeHookExec(
+	raw DecodeHookFunc,
+	from reflect.Value, to reflect.Value) (interface{}, error) {
+
+	switch f := typedDecodeHook(raw).(type) {
+	case DecodeHookFuncType:
+		return f(from.Type(), to.Type(), from.Interface())
+	case DecodeHookFuncKind:
+		return f(from.Kind(), to.Kind(), from.Interface())
+	case DecodeHookFuncValue:
+		return f(from, to)
+	default:
+		return nil, errors.New("invalid decode hook signature")
+	}
+}
+
+// ComposeDecodeHookFunc creates a single DecodeHookFunc that
+// automatically composes multiple DecodeHookFuncs.
+//
+// The composed funcs are called in order, with the result of the
+// previous transformation.
+func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc {
+	return func(f reflect.Value, t reflect.Value) (interface{}, error) {
+		var err error
+		var data interface{}
+		newFrom := f
+		for _, f1 := range fs {
+			data, err = DecodeHookExec(f1, newFrom, t)
+			if err != nil {
+				return nil, err
+			}
+			newFrom = reflect.ValueOf(data)
+		}
+
+		return data, nil
+	}
+}
+
+// StringToSliceHookFunc returns a DecodeHookFunc that converts
+// string to []string by splitting on the given sep.
+func StringToSliceHookFunc(sep string) DecodeHookFunc {
+	return func(
+		f reflect.Kind,
+		t reflect.Kind,
+		data interface{}) (interface{}, error) {
+		if f != reflect.String || t != reflect.Slice {
+			return data, nil
+		}
+
+		raw := data.(string)
+		if raw == "" {
+			return []string{}, nil
+		}
+
+		return strings.Split(raw, sep), nil
+	}
+}
+
+// StringToTimeDurationHookFunc returns a DecodeHookFunc that converts
+// strings to time.Duration.
+func StringToTimeDurationHookFunc() DecodeHookFunc {
+	return func(
+		f reflect.Type,
+		t reflect.Type,
+		data interface{}) (interface{}, error) {
+		if f.Kind() != reflect.String {
+			return data, nil
+		}
+		if t != reflect.TypeOf(time.Duration(5)) {
+			return data, nil
+		}
+
+		// Convert it by parsing
+		return time.ParseDuration(data.(string))
+	}
+}
+
+// StringToIPHookFunc returns a DecodeHookFunc that converts
+// strings to net.IP
+func StringToIPHookFunc() DecodeHookFunc {
+	return func(
+		f reflect.Type,
+		t reflect.Type,
+		data interface{}) (interface{}, error) {
+		if f.Kind() != reflect.String {
+			return data, nil
+		}
+		if t != reflect.TypeOf(net.IP{}) {
+			return data, nil
+		}
+
+		// Convert it by parsing
+		ip := net.ParseIP(data.(string))
+		if ip == nil {
+			return net.IP{}, fmt.Errorf("failed parsing ip %v", data)
+		}
+
+		return ip, nil
+	}
+}
+
+// StringToIPNetHookFunc returns a DecodeHookFunc that converts
+// strings to net.IPNet
+func StringToIPNetHookFunc() DecodeHookFunc {
+	return func(
+		f reflect.Type,
+		t reflect.Type,
+		data interface{}) (interface{}, error) {
+		if f.Kind() != reflect.String {
+			return data, nil
+		}
+		if t != reflect.TypeOf(net.IPNet{}) {
+			return data, nil
+		}
+
+		// Convert it by parsing
+		_, net, err := net.ParseCIDR(data.(string))
+		return net, err
+	}
+}
+
+// StringToTimeHookFunc returns a DecodeHookFunc that converts
+// strings to time.Time.
+func StringToTimeHookFunc(layout string) DecodeHookFunc {
+	return func(
+		f reflect.Type,
+		t reflect.Type,
+		data interface{}) (interface{}, error) {
+		if f.Kind() != reflect.String {
+			return data, nil
+		}
+		if t != reflect.TypeOf(time.Time{}) {
+			return data, nil
+		}
+
+		// Convert it by parsing
+		return time.Parse(layout, data.(string))
+	}
+}
+
+// WeaklyTypedHook is a DecodeHookFunc which adds support for weak typing to
+// the decoder.
+//
+// Note that this is significantly different from the WeaklyTypedInput option
+// of the DecoderConfig.
+func WeaklyTypedHook(
+	f reflect.Kind,
+	t reflect.Kind,
+	data interface{}) (interface{}, error) {
+	dataVal := reflect.ValueOf(data)
+	switch t {
+	case reflect.String:
+		switch f {
+		case reflect.Bool:
+			if dataVal.Bool() {
+				return "1", nil
+			}
+			return "0", nil
+		case reflect.Float32:
+			return strconv.FormatFloat(dataVal.Float(), 'f', -1, 64), nil
+		case reflect.Int:
+			return strconv.FormatInt(dataVal.Int(), 10), nil
+		case reflect.Slice:
+			dataType := dataVal.Type()
+			elemKind := dataType.Elem().Kind()
+			if elemKind == reflect.Uint8 {
+				return string(dataVal.Interface().([]uint8)), nil
+			}
+		case reflect.Uint:
+			return strconv.FormatUint(dataVal.Uint(), 10), nil
+		}
+	}
+
+	return data, nil
+}
+
+func RecursiveStructToMapHookFunc() DecodeHookFunc {
+	return func(f reflect.Value, t reflect.Value) (interface{}, error) {
+		if f.Kind() != reflect.Struct {
+			return f.Interface(), nil
+		}
+
+		var i interface{} = struct{}{}
+		if t.Type() != reflect.TypeOf(&i).Elem() {
+			return f.Interface(), nil
+		}
+
+		m := make(map[string]interface{})
+		t.Set(reflect.ValueOf(m))
+
+		return f.Interface(), nil
+	}
+}
+
+// TextUnmarshallerHookFunc returns a DecodeHookFunc that applies
+// strings to the UnmarshalText function, when the target type
+// implements the encoding.TextUnmarshaler interface
+func TextUnmarshallerHookFunc() DecodeHookFuncType {
+	return func(
+		f reflect.Type,
+		t reflect.Type,
+		data interface{}) (interface{}, error) {
+		if f.Kind() != reflect.String {
+			return data, nil
+		}
+		result := reflect.New(t).Interface()
+		unmarshaller, ok := result.(encoding.TextUnmarshaler)
+		if !ok {
+			return data, nil
+		}
+		if err := unmarshaller.UnmarshalText([]byte(data.(string))); err != nil {
+			return nil, err
+		}
+		return result, nil
+	}
+}

vendor/github.com/mitchellh/mapstructure/error.go

@@ -0,0 +1,50 @@
+package mapstructure
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// Error implements the error interface and can represents multiple
+// errors that occur in the course of a single decode.
+type Error struct {
+	Errors []string
+}
+
+func (e *Error) Error() string {
+	points := make([]string, len(e.Errors))
+	for i, err := range e.Errors {
+		points[i] = fmt.Sprintf("* %s", err)
+	}
+
+	sort.Strings(points)
+	return fmt.Sprintf(
+		"%d error(s) decoding:\n\n%s",
+		len(e.Errors), strings.Join(points, "\n"))
+}
+
+// WrappedErrors implements the errwrap.Wrapper interface to make this
+// return value more useful with the errwrap and go-multierror libraries.
+func (e *Error) WrappedErrors() []error {
+	if e == nil {
+		return nil
+	}
+
+	result := make([]error, len(e.Errors))
+	for i, e := range e.Errors {
+		result[i] = errors.New(e)
+	}
+
+	return result
+}
+
+func appendErrors(errors []string, err error) []string {
+	switch e := err.(type) {
+	case *Error:
+		return append(errors, e.Errors...)
+	default:
+		return append(errors, e.Error())
+	}
+}

vendor/github.com/mitchellh/mapstructure/go.mod

@@ -0,0 +1,3 @@
+module github.com/mitchellh/mapstructure
+
+go 1.14

vendor/modules.txt

@@ -338,6 +338,9 @@ github.com/mattn/go-isatty
 ## explicit
 github.com/microcosm-cc/bluemonday
 github.com/microcosm-cc/bluemonday/css
+# github.com/mitchellh/mapstructure v1.4.1
+## explicit
+github.com/mitchellh/mapstructure
 # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
 ## explicit
 github.com/modern-go/concurrent