mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

Search (#36) 

First implementation of search functionality for remote account and status lookups.
This commit is contained in:
Tobi Smethurst 2021-05-29 19:39:43 +02:00 committed by GitHub
parent cb54324430
commit 1fe5e36ac3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 769 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
PROGRESS.md
View File

@ -100,7 +100,7 @@
* [ ] Timelines
* [ ] /api/v1/timelines/public GET (See the public/federated timeline)
* [ ] /api/v1/timelines/tag/:hashtag GET (Get public statuses that use hashtag)
* [ ] /api/v1/timelines/home GET (View statuses from followed users)
* [x] /api/v1/timelines/home GET (View statuses from followed users)
* [ ] /api/v1/timelines/list/:list_id GET (Get statuses in given list)
* [ ] Conversations
* [ ] /api/v1/conversations GET (Get a list of direct message convos)
@ -121,8 +121,8 @@
* [ ] Streaming
* [ ] /api/v1/streaming WEBSOCKETS (Stream live events to user via websockets)
* [ ] Notifications
* [ ] /api/v1/notifications GET (Get list of notifications)
* [ ] /api/v1/notifications/:id GET (Get a single notification)
* [x] /api/v1/notifications GET (Get list of notifications)
* [x] /api/v1/notifications/:id GET (Get a single notification)
* [ ] /api/v1/notifications/clear POST (Clear all notifications)
* [ ] /api/v1/notifications/:id POST (Clear a single notification)
* [ ] Push
@ -130,8 +130,8 @@
* [ ] /api/v1/push/subscription GET (Get current subscription)
* [ ] /api/v1/push/subscription PUT (Change notification types)
* [ ] /api/v1/push/subscription DELETE (Delete current subscription)
* [ ] Search
* [ ] /api/v2/search GET (Get search query results)
* [x] Search
* [x] /api/v2/search GET (Get search query results)
* [ ] Instance
* [x] /api/v1/instance GET (Get instance information)
* [ ] /api/v1/instance PATCH (Update instance information)
@ -174,7 +174,7 @@
* [ ] Federation modes
* [ ] 'Slow' federation
* [ ] Reputation scoring system for instances
* [ ] 'Greedy' federation
* [x] 'Greedy' federation
* [ ] No federation (insulate this instance from the Fediverse)
* [ ] Allowlist
* [x] Secure HTTP signatures (creation and validation)

88
internal/api/client/search/search.go Normal file
View File

@ -0,0 +1,88 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package search
import (
"net/http"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/api"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/message"
"github.com/superseriousbusiness/gotosocial/internal/router"
)
const (
// BasePath is the base path for serving v1 of the search API
BasePathV1 = "/api/v1/search"
// BasePathV2 is the base path for serving v2 of the search API
BasePathV2 = "/api/v2/search"
// AccountIDKey -- If provided, statuses returned will be authored only by this account
AccountIDKey = "account_id"
// MaxIDKey -- Return results older than this id
MaxIDKey = "max_id"
// MinIDKey -- Return results immediately newer than this id
MinIDKey = "min_id"
// TypeKey -- Enum(accounts, hashtags, statuses)
TypeKey = "type"
// ExcludeUnreviewedKey -- Filter out unreviewed tags? Defaults to false. Use true when trying to find trending tags.
ExcludeUnreviewedKey = "exclude_unreviewed"
// QueryKey -- The search query
QueryKey = "q"
// ResolveKey -- Attempt WebFinger lookup. Defaults to false.
ResolveKey = "resolve"
// LimitKey -- Maximum number of results to load, per type. Defaults to 20. Max 40.
LimitKey = "limit"
// OffsetKey -- Offset in search results. Used for pagination. Defaults to 0.
OffsetKey = "offset"
// FollowingKey -- Only include accounts that the user is following. Defaults to false.
FollowingKey = "following"
// TypeAccounts --
TypeAccounts = "accounts"
// TypeHashtags --
TypeHashtags = "hashtags"
// TypeStatuses --
TypeStatuses = "statuses"
)
// Module implements the ClientAPIModule interface for everything related to searching
type Module struct {
config *config.Config
processor message.Processor
log *logrus.Logger
}
// New returns a new search module
func New(config *config.Config, processor message.Processor, log *logrus.Logger) api.ClientModule {
return &Module{
config: config,
processor: processor,
log: log,
}
}
// Route attaches all routes from this module to the given router
func (m *Module) Route(r router.Router) error {
r.AttachHandler(http.MethodGet, BasePathV1, m.SearchGETHandler)
r.AttachHandler(http.MethodGet, BasePathV2, m.SearchGETHandler)
return nil
}

151
internal/api/client/search/searchget.go Normal file
View File

@ -0,0 +1,151 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package search
import (
"fmt"
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
)
// SearchGETHandler handles searches for local and remote accounts, statuses, and hashtags.
// It corresponds to the mastodon endpoint described here: https://docs.joinmastodon.org/methods/search/
func (m *Module) SearchGETHandler(c *gin.Context) {
l := m.log.WithFields(logrus.Fields{
"func": "SearchGETHandler",
"request_uri": c.Request.RequestURI,
"user_agent": c.Request.UserAgent(),
"origin_ip": c.ClientIP(),
})
l.Debugf("entering function")
authed, err := oauth.Authed(c, true, true, true, true) // we don't really need an app here but we want everything else
if err != nil {
l.Errorf("error authing search request: %s", err)
c.JSON(http.StatusBadRequest, gin.H{"error": "not authed"})
return
}
accountID := c.Query(AccountIDKey)
maxID := c.Query(MaxIDKey)
minID := c.Query(MinIDKey)
searchType := c.Query(TypeKey)
excludeUnreviewed := false
excludeUnreviewedString := c.Query(ExcludeUnreviewedKey)
if excludeUnreviewedString != "" {
var err error
excludeUnreviewed, err = strconv.ParseBool(excludeUnreviewedString)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", excludeUnreviewedString, err)})
return
}
}
query := c.Query(QueryKey)
if query == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "query parameter q was empty"})
return
}
resolve := false
resolveString := c.Query(ResolveKey)
if resolveString != "" {
var err error
resolve, err = strconv.ParseBool(resolveString)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", resolveString, err)})
return
}
}
limit := 20
limitString := c.Query(LimitKey)
if limitString != "" {
i, err := strconv.ParseInt(limitString, 10, 64)
if err != nil {
l.Debugf("error parsing limit string: %s", err)
c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse limit query param"})
return
}
limit = int(i)
}
if limit > 40 {
limit = 40
}
if limit < 1 {
limit = 1
}
offset := 0
offsetString := c.Query(OffsetKey)
if offsetString != "" {
i, err := strconv.ParseInt(offsetString, 10, 64)
if err != nil {
l.Debugf("error parsing offset string: %s", err)
c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse offset query param"})
return
}
offset = int(i)
}
if limit > 40 {
limit = 40
}
if limit < 1 {
limit = 1
}
following := false
followingString := c.Query(FollowingKey)
if followingString != "" {
var err error
following, err = strconv.ParseBool(followingString)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", followingString, err)})
return
}
}
searchQuery := &model.SearchQuery{
AccountID: accountID,
MaxID: maxID,
MinID: minID,
Type: searchType,
ExcludeUnreviewed: excludeUnreviewed,
Query: query,
Resolve: resolve,
Limit: limit,
Offset: offset,
Following: following,
}
results, errWithCode := m.processor.SearchGet(authed, searchQuery)
if errWithCode != nil {
l.Debugf("error searching: %s", errWithCode.Error())
c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
return
}
c.JSON(http.StatusOK, results)
}

52
internal/api/model/search.go Normal file
View File

@ -0,0 +1,52 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package model
// SearchQuery corresponds to search parameters as submitted through the client API.
// See https://docs.joinmastodon.org/methods/search/
type SearchQuery struct {
// If provided, statuses returned will be authored only by this account
AccountID string
// Return results older than this id
MaxID string
// Return results immediately newer than this id
MinID string
// Enum(accounts, hashtags, statuses)
Type string
// Filter out unreviewed tags? Defaults to false. Use true when trying to find trending tags.
ExcludeUnreviewed bool
// The search query
Query string
// Attempt WebFinger lookup. Defaults to false.
Resolve bool
// Maximum number of results to load, per type. Defaults to 20. Max 40.
Limit int
// Offset in search results. Used for pagination. Defaults to 0.
Offset int
// Only include accounts that the user is following. Defaults to false.
Following bool
}
// SearchResult corresponds to a search result, containing accounts, statuses, and hashtags.
// See https://docs.joinmastodon.org/methods/search/
type SearchResult struct {
Accounts []Account `json:"accounts"`
Statuses []Status `json:"statuses"`
Hashtags []Tag `json:"hashtags"`
}

5
internal/db/db.go
View File

@ -45,8 +45,9 @@ func (e ErrAlreadyExists) Error() string {
}
type Where struct {
Key string
Value interface{}
Key string
Value interface{}
CaseInsensitive bool
}
// DB provides methods for interacting with an underlying database or other storage mechanism (for now, just postgres).

8
internal/db/pg/pg.go
View File

@ -223,7 +223,12 @@ func (ps *postgresService) GetWhere(where []db.Where, i interface{}) error {
q := ps.conn.Model(i)
for _, w := range where {
q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
if w.CaseInsensitive {
q = q.Where("LOWER(?) = LOWER(?)", pg.Safe(w.Key), w.Value)
} else {
q = q.Where("? = ?", pg.Safe(w.Key), w.Value)
}
}
if err := q.Select(); err != nil {
@ -1143,7 +1148,6 @@ func (ps *postgresService) GetNotificationsForAccount(accountID string, limit in
q := ps.conn.Model(&notifications).Where("target_account_id = ?", accountID)
if maxID != "" {
n := &gtsmodel.Notification{}
if err := ps.conn.Model(n).Where("id = ?", maxID).Select(); err != nil {

7
internal/federation/commonbehavior.go
View File

@ -25,6 +25,7 @@ import (
"net/url"
"github.com/go-fed/activity/pub"
"github.com/go-fed/activity/streams"
"github.com/go-fed/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/util"
@ -59,7 +60,7 @@ import (
func (f *federator) AuthenticateGetInbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (context.Context, bool, error) {
// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
// the CLIENT API, not through the federation API, so we just do nothing here.
return nil, false, nil
return ctx, false, nil
}
// AuthenticateGetOutbox delegates the authentication of a GET to an
@ -84,7 +85,7 @@ func (f *federator) AuthenticateGetInbox(ctx context.Context, w http.ResponseWri
func (f *federator) AuthenticateGetOutbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (context.Context, bool, error) {
// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
// the CLIENT API, not through the federation API, so we just do nothing here.
return nil, false, nil
return ctx, false, nil
}
// GetOutbox returns the OrderedCollection inbox of the actor for this
@ -98,7 +99,7 @@ func (f *federator) AuthenticateGetOutbox(ctx context.Context, w http.ResponseWr
func (f *federator) GetOutbox(ctx context.Context, r *http.Request) (vocab.ActivityStreamsOrderedCollectionPage, error) {
// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
// the CLIENT API, not through the federation API, so we just do nothing here.
return nil, nil
return streams.NewActivityStreamsOrderedCollectionPage(), nil
}
// NewTransport returns a new Transport on behalf of a specific actor.

14
internal/federation/federatingdb/followers.go
View File

@ -10,6 +10,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// Followers obtains the Followers Collection for an actor with the
@ -28,8 +29,17 @@ func (f *federatingDB) Followers(c context.Context, actorIRI *url.URL) (follower
l.Debugf("entering FOLLOWERS function with actorIRI %s", actorIRI.String())
acct := &gtsmodel.Account{}
if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
if util.IsUserPath(actorIRI) {
if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
}
} else if util.IsFollowersPath(actorIRI) {
if err := f.db.GetWhere([]db.Where{{Key: "followers_uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with followers uri %s: %s", actorIRI.String(), err)
}
} else {
return nil, fmt.Errorf("could not parse actor IRI %s as users or followers path", actorIRI.String())
}
acctFollowers := []gtsmodel.Follow{}

13
internal/federation/federatingdb/following.go
View File

@ -10,6 +10,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// Following obtains the Following Collection for an actor with the
@ -28,8 +29,16 @@ func (f *federatingDB) Following(c context.Context, actorIRI *url.URL) (followin
l.Debugf("entering FOLLOWING function with actorIRI %s", actorIRI.String())
acct := &gtsmodel.Account{}
if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
if util.IsUserPath(actorIRI) {
if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err)
}
} else if util.IsFollowingPath(actorIRI) {
if err := f.db.GetWhere([]db.Where{{Key: "following_uri", Value: actorIRI.String()}}, acct); err != nil {
return nil, fmt.Errorf("db error getting account with following uri %s: %s", actorIRI.String(), err)
}
} else {
return nil, fmt.Errorf("could not parse actor IRI %s as users or following path", actorIRI.String())
}
acctFollowing := []gtsmodel.Follow{}

7
internal/federation/federatingdb/lock.go
View File

@ -42,6 +42,10 @@ func (f *federatingDB) Lock(c context.Context, id *url.URL) error {
// Strategy: create a new lock, if stored, continue. Otherwise, lock the
// existing mutex.
if id == nil {
return errors.New("Lock: id was nil")
}
mu := &sync.Mutex{}
mu.Lock() // Optimistically lock if we do store it.
i, loaded := f.locks.LoadOrStore(id.String(), mu)
@ -59,6 +63,9 @@ func (f *federatingDB) Lock(c context.Context, id *url.URL) error {
func (f *federatingDB) Unlock(c context.Context, id *url.URL) error {
// Once Go-Fed is done calling Database methods, the relevant `id`
// entries are unlocked.
if id == nil {
return errors.New("Unlock: id was nil")
}
i, ok := f.locks.Load(id.String())
if !ok {

5
internal/federation/federatingprotocol.go
View File

@ -26,6 +26,7 @@ import (
"net/url"
"github.com/go-fed/activity/pub"
"github.com/go-fed/activity/streams"
"github.com/go-fed/activity/streams/vocab"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -310,7 +311,7 @@ func (f *federator) MaxDeliveryRecursionDepth(ctx context.Context) int {
// logic to be used, but the implementation must not modify it.
func (f *federator) FilterForwarding(ctx context.Context, potentialRecipients []*url.URL, a pub.Activity) ([]*url.URL, error) {
// TODO
return nil, nil
return []*url.URL{}, nil
}
// GetInbox returns the OrderedCollection inbox of the actor for this
@ -324,5 +325,5 @@ func (f *federator) FilterForwarding(ctx context.Context, potentialRecipients []
func (f *federator) GetInbox(ctx context.Context, r *http.Request) (vocab.ActivityStreamsOrderedCollectionPage, error) {
// IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through
// the CLIENT API, not through the federation API, so we just do nothing here.
return nil, nil
return streams.NewActivityStreamsOrderedCollectionPage(), nil
}

3
internal/federation/federator.go
View File

@ -40,6 +40,9 @@ type Federator interface {
// AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources.
// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
AuthenticateFederatedRequest(username string, r *http.Request) (*url.URL, error)
// FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that
// account, or an error if it doesn't exist or can't be retrieved.
FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error)
// DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI).
// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error)

69
internal/federation/finger.go Normal file
View File

@ -0,0 +1,69 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package federation
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
)
func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) {
t, err := f.GetTransportForUser(requestingUsername)
if err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
}
b, err := t.Finger(context.Background(), targetUsername, targetDomain)
if err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: error doing request on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
}
resp := &apimodel.WebfingerAccountResponse{}
if err := json.Unmarshal(b, resp); err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
}
if len(resp.Links) == 0 {
return nil, fmt.Errorf("FingerRemoteAccount: no links found in webfinger response %s", string(b))
}
// look through the links for the first one that matches "application/activity+json", this is what we need
for _, l := range resp.Links {
if strings.EqualFold(l.Type, "application/activity+json") {
if l.Href == "" || l.Rel != "self" {
continue
}
accountURI, err := url.Parse(l.Href)
if err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: couldn't parse url %s: %s", l.Href, err)
}
// found it!
return accountURI, nil
}
}
return nil, errors.New("FingerRemoteAccount: no match found in webfinger response")
}

3
internal/gotosocial/actions.go
View File

@ -38,6 +38,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
mediaModule "github.com/superseriousbusiness/gotosocial/internal/api/client/media"
"github.com/superseriousbusiness/gotosocial/internal/api/client/notification"
"github.com/superseriousbusiness/gotosocial/internal/api/client/search"
"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
"github.com/superseriousbusiness/gotosocial/internal/api/client/timeline"
"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
@ -122,6 +123,7 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
usersModule := user.New(c, processor, log)
timelineModule := timeline.New(c, processor, log)
notificationModule := notification.New(c, processor, log)
searchModule := search.New(c, processor, log)
mm := mediaModule.New(c, processor, log)
fileServerModule := fileserver.New(c, processor, log)
adminModule := admin.New(c, processor, log)
@ -146,6 +148,7 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
usersModule,
timelineModule,
notificationModule,
searchModule,
}
for _, m := range apis {

6
internal/message/fromfederatorprocess.go
View File

@ -105,7 +105,9 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
}
if err := p.db.Put(incomingAnnounce); err != nil {
return fmt.Errorf("error adding dereferenced announce to the db: %s", err)
if _, ok := err.(db.ErrAlreadyExists); !ok {
return fmt.Errorf("error adding dereferenced announce to the db: %s", err)
}
}
if err := p.notifyAnnounce(incomingAnnounce); err != nil {
@ -407,7 +409,7 @@ func (p *processor) dereferenceAnnounce(announce *gtsmodel.Status, requestingUse
}
// now dereference additional fields straight away (we're already async here so we have time)
if err := p.dereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
if err := p.dereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
}

3
internal/message/processor.go
View File

@ -109,6 +109,9 @@ type Processor interface {
// NotificationsGet
NotificationsGet(authed *oauth.Auth, limit int, maxID string) ([]*apimodel.Notification, ErrorWithCode)
// SearchGet performs a search with the given params, resolving/dereferencing remotely as desired
SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode)
// StatusCreate processes the given form to create a new status, returning the api model representation of that status if it's OK.
StatusCreate(authed *oauth.Auth, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, error)
// StatusDelete processes the delete of a given status, returning the deleted status if the delete goes through.

292
internal/message/searchprocess.go Normal file
View File

@ -0,0 +1,292 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package message
import (
"errors"
"net/url"
"strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode) {
results := &apimodel.SearchResult{
Accounts: []apimodel.Account{},
Statuses: []apimodel.Status{},
Hashtags: []apimodel.Tag{},
}
foundAccounts := []*gtsmodel.Account{}
foundStatuses := []*gtsmodel.Status{}
// foundHashtags := []*gtsmodel.Tag{}
// convert the query to lowercase and trim leading/trailing spaces
query := strings.ToLower(strings.TrimSpace(searchQuery.Query))
// check if the query is a URI and just do a lookup for that, straight up
if uri, err := url.Parse(query); err == nil {
// 1. check if it's a status
foundStatus, err := p.searchStatusByURI(authed, uri, searchQuery.Resolve)
if err != nil {
return nil, NewErrorInternalError(err)
}
if foundStatus != nil {
foundStatuses = append(foundStatuses, foundStatus)
}
// 2. check if it's an account
foundAccount, err := p.searchAccountByURI(authed, uri, searchQuery.Resolve)
if err != nil {
return nil, NewErrorInternalError(err)
}
if foundAccount != nil {
foundAccounts = append(foundAccounts, foundAccount)
}
}
// check if the query is something like @whatever_username@example.org -- this means it's a remote account
if util.IsMention(searchQuery.Query) {
foundAccount, err := p.searchAccountByMention(authed, searchQuery.Query, searchQuery.Resolve)
if err != nil {
return nil, NewErrorInternalError(err)
}
if foundAccount != nil {
foundAccounts = append(foundAccounts, foundAccount)
}
}
/*
FROM HERE ON we have our search results, it's just a matter of filtering them according to what this user is allowed to see,
and then converting them into our frontend format.
*/
for _, foundAccount := range foundAccounts {
// make sure there's no block in either direction between the account and the requester
if blocked, err := p.db.Blocked(authed.Account.ID, foundAccount.ID); err == nil && !blocked {
// all good, convert it and add it to the results
acctMasto, err := p.tc.AccountToMastoPublic(foundAccount)
if err != nil {
return nil, NewErrorInternalError(err)
}
results.Accounts = append(results.Accounts, *acctMasto)
}
}
for _, foundStatus := range foundStatuses {
statusOwner := &gtsmodel.Account{}
if err := p.db.GetByID(foundStatus.AccountID, statusOwner); err != nil {
continue
}
relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(foundStatus)
if err != nil {
continue
}
if visible, err := p.db.StatusVisible(foundStatus, statusOwner, authed.Account, relevantAccounts); !visible || err != nil {
continue
}
statusMasto, err := p.tc.StatusToMasto(foundStatus, statusOwner, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, nil)
if err != nil {
continue
}
results.Statuses = append(results.Statuses, *statusMasto)
}
return results, nil
}
func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (foundStatus *gtsmodel.Status, err error) {
// 1. check if it's a status
maybeStatus := &gtsmodel.Status{}
if err = p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
// we have it and it's a status
foundStatus = maybeStatus
return
} else if err = p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
// we have it and it's a status
foundStatus = maybeStatus
return
}
// we don't have it locally so dereference it if we're allowed to
if resolve {
statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri)
if err == nil {
// it IS a status!
// extract the status owner's IRI from the statusable
var statusOwnerURI *url.URL
statusAttributedTo := statusable.GetActivityStreamsAttributedTo()
for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() {
if i.IsIRI() {
statusOwnerURI = i.GetIRI()
break
}
}
if statusOwnerURI == nil {
return nil, NewErrorInternalError(errors.New("couldn't extract ownerAccountURI from statusable"))
}
// make sure the status owner exists in the db by searching for it
_, err := p.searchAccountByURI(authed, statusOwnerURI, resolve)
if err != nil {
return nil, err
}
// we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly
// first turn it into a gtsmodel.Status
status, err := p.tc.ASStatusToStatus(statusable)
if err != nil {
return nil, NewErrorInternalError(err)
}
// put it in the DB so it gets a UUID
if err := p.db.Put(status); err != nil {
return nil, NewErrorInternalError(err)
}
// properly dereference everything in the status (media attachments etc)
if err := p.dereferenceStatusFields(status, authed.Account.Username); err != nil {
return nil, NewErrorInternalError(err)
}
// update with the nicely dereferenced status
if err := p.db.UpdateByID(status.ID, status); err != nil {
return nil, NewErrorInternalError(err)
}
foundStatus = status
}
}
return
}
func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (foundAccount *gtsmodel.Account, err error) {
maybeAccount := &gtsmodel.Account{}
if err = p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil {
// we have it and it's an account
foundAccount = maybeAccount
return
} else if err = p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil {
// we have it and it's an account
foundAccount = maybeAccount
return
}
if resolve {
// we don't have it locally so try and dereference it
accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri)
if err == nil {
// it IS an account!
account, err := p.tc.ASRepresentationToAccount(accountable, false)
if err != nil {
return nil, NewErrorInternalError(err)
}
if err := p.db.Put(account); err != nil {
return nil, NewErrorInternalError(err)
}
if err := p.dereferenceAccountFields(account, authed.Account.Username, false); err != nil {
return nil, NewErrorInternalError(err)
}
foundAccount = account
}
}
return
}
func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, resolve bool) (foundAccount *gtsmodel.Account, err error) {
// query is for a remote account
username, domain, err := util.ExtractMentionParts(mention)
if err != nil {
return nil, NewErrorBadRequest(err)
}
// if it's a local account we can skip a whole bunch of stuff
maybeAcct := &gtsmodel.Account{}
if domain == p.config.Host {
if err = p.db.GetLocalAccountByUsername(username, maybeAcct); err != nil {
return
}
foundAccount = maybeAcct
return
}
// it's not a local account so first we'll check if it's in the database already...
where := []db.Where{
{Key: "username", Value: username, CaseInsensitive: true},
{Key: "domain", Value: domain, CaseInsensitive: true},
}
err = p.db.GetWhere(where, maybeAcct)
if err == nil {
// we've got it stored locally already!
foundAccount = maybeAcct
return
}
if _, ok := err.(db.ErrNoEntries); !ok {
// if it's not errNoEntries there's been a real database error so bail at this point
return nil, NewErrorInternalError(err)
}
// we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it
if resolve {
// we're allowed to resolve it so let's try
// first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account
acctURI, err := p.federator.FingerRemoteAccount(authed.Account.Username, username, domain)
if err != nil {
// something went wrong doing the webfinger lookup so we can't process the request
return nil, NewErrorInternalError(err)
}
// dereference the account based on the URI we retrieved from the webfinger lookup
accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI)
if err != nil {
// something went wrong doing the dereferencing so we can't process the request
return nil, NewErrorInternalError(err)
}
// convert the dereferenced account to the gts model of that account
foundAccount, err = p.tc.ASRepresentationToAccount(accountable, false)
if err != nil {
// something went wrong doing the conversion to a gtsmodel.Account so we can't process the request
return nil, NewErrorInternalError(err)
}
// put this new account in our database
if err := p.db.Put(foundAccount); err != nil {
return nil, NewErrorInternalError(err)
}
// properly dereference all the fields on the account immediately
if err := p.dereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil {
return nil, NewErrorInternalError(err)
}
}
return
}

41
internal/transport/transport.go
View File

@ -19,6 +19,8 @@ import (
type Transport interface {
pub.Transport
DereferenceMedia(c context.Context, iri *url.URL, expectedContentType string) ([]byte, error)
// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
Finger(c context.Context, targetUsername string, targetDomains string) ([]byte, error)
}
// transport implements the Transport interface
@ -83,3 +85,42 @@ func (t *transport) DereferenceMedia(c context.Context, iri *url.URL, expectedCo
}
return ioutil.ReadAll(resp.Body)
}
func (t *transport) Finger(c context.Context, targetUsername string, targetDomain string) ([]byte, error) {
l := t.log.WithField("func", "Finger")
urlString := fmt.Sprintf("https://%s/.well-known/webfinger?resource=acct:%s@%s", targetDomain, targetUsername, targetDomain)
l.Debugf("performing GET to %s", urlString)
iri, err := url.Parse(urlString)
if err != nil {
return nil, fmt.Errorf("Finger: error parsing url %s: %s", urlString, err)
}
l.Debugf("performing GET to %s", iri.String())
req, err := http.NewRequest("GET", iri.String(), nil)
if err != nil {
return nil, err
}
req = req.WithContext(c)
req.Header.Add("Accept", "application/json")
req.Header.Add("Accept", "application/jrd+json")
req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
req.Header.Set("Host", iri.Host)
t.getSignerMu.Lock()
err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
t.getSignerMu.Unlock()
if err != nil {
return nil, err
}
resp, err := t.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
}
return ioutil.ReadAll(resp.Body)
}

2
internal/typeutils/converter.go
View File

@ -145,7 +145,7 @@ type TypeConverter interface {
/*
WRAPPER CONVENIENCE FUNCTIONS
*/
// WrapPersonInUpdate
WrapPersonInUpdate(person vocab.ActivityStreamsPerson, originAccount *gtsmodel.Account) (vocab.ActivityStreamsUpdate, error)
}

8
internal/typeutils/internal.go
View File

@ -40,10 +40,10 @@ func (c *converter) StatusToBoost(s *gtsmodel.Status, boostingAccount *gtsmodel.
URL: boostWrapperStatusURL,
// the boosted status is not created now, but the boost certainly is
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
Local: local,
AccountID: boostingAccount.ID,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
Local: local,
AccountID: boostingAccount.ID,
// replies can be boosted, but boosts are never replies
InReplyToID: "",

1
internal/typeutils/internaltoas.go
View File

@ -252,6 +252,7 @@ func (c *converter) AccountToAS(a *gtsmodel.Account) (vocab.ActivityStreamsPerso
headerImage.SetActivityStreamsUrl(headerURLProperty)
headerProperty.AppendActivityStreamsImage(headerImage)
person.SetActivityStreamsImage(headerProperty)
}
return person, nil

5
internal/util/statustools.go
View File

@ -77,6 +77,11 @@ func ExtractMentionParts(mention string) (username, domain string, err error) {
return
}
// IsMention returns true if the passed string looks like @whatever@example.org
func IsMention(mention string) bool {
return mentionNameRegex.MatchString(strings.ToLower(mention))
}
// unique returns a deduplicated version of a given string slice.
func unique(s []string) []string {
keys := make(map[string]bool)