From 0f2de6394a1c52d47e326bb7d7d129a217ae4f6f Mon Sep 17 00:00:00 2001 From: Tobi Smethurst <31960611+tsmethurst@users.noreply.github.com> Date: Tue, 10 Aug 2021 13:32:39 +0200 Subject: [PATCH] Dereference remote replies (#132) * decided where to put reply dereferencing * fiddling with dereferencing threads * further adventures * tidy up some stuff * move dereferencing functionality * a bunch of refactoring * go fmt * more refactoring * bleep bloop * docs and linting * start implementing replies collection on gts side * fiddling around * allow dereferencing our replies * lint, fmt --- docs/api/swagger.yaml | 119 +++- .../diagrams/conversation_thread.drawio | 1 + docs/assets/diagrams/conversation_thread.png | Bin 0 -> 23930 bytes .../behaviors/conversation_threads.md | 45 ++ docs/federation/glossary.md | 27 + docs/federation/principles.md | 9 + docs/federation/security.md | 7 + .../asextractionutil.go => ap/extract.go} | 128 +++-- internal/ap/interfaces.go | 321 +++++++++++ .../api/client/account/accountupdate_test.go | 6 +- .../api/client/fileserver/servefile_test.go | 4 +- internal/api/client/media/mediacreate_test.go | 4 +- .../api/client/status/statusboost_test.go | 4 +- .../api/client/status/statuscreate_test.go | 4 +- internal/api/client/status/statusfave_test.go | 4 +- .../api/client/status/statusfavedby_test.go | 4 +- internal/api/client/status/statusget_test.go | 4 +- .../api/client/status/statusunfave_test.go | 4 +- internal/api/s2s/user/repliesget.go | 186 +++++++ internal/api/s2s/user/repliesget_test.go | 241 ++++++++ internal/api/s2s/user/user.go | 10 + internal/api/s2s/user/user_test.go | 16 +- internal/api/s2s/user/userget_test.go | 66 +-- internal/cliactions/server/server.go | 2 +- internal/cliactions/testrig/testrig.go | 4 +- internal/db/db.go | 8 +- internal/db/pg/statuscontext.go | 31 +- internal/federation/authenticate.go | 26 +- internal/federation/dereference.go | 518 +----------------- internal/federation/dereferencing/account.go | 243 ++++++++ internal/federation/dereferencing/announce.go | 65 +++ internal/federation/dereferencing/blocked.go | 41 ++ .../dereferencing/collectionpage.go | 70 +++ .../federation/dereferencing/dereferencer.go | 73 +++ .../federation/dereferencing/handshake.go | 98 ++++ internal/federation/dereferencing/instance.go | 40 ++ internal/federation/dereferencing/status.go | 369 +++++++++++++ internal/federation/dereferencing/thread.go | 250 +++++++++ internal/federation/federatingdb/update.go | 4 +- internal/federation/federatingprotocol.go | 51 +- internal/federation/federator.go | 41 +- internal/federation/federator_test.go | 6 +- internal/federation/finger.go | 2 +- internal/federation/handshake.go | 75 +-- internal/federation/transport.go | 32 +- internal/gtsmodel/activitystreams.go | 4 + internal/gtsmodel/status.go | 4 +- internal/processing/account/get.go | 9 - internal/processing/account/getfollowers.go | 6 - internal/processing/account/getfollowing.go | 6 - internal/processing/federation.go | 199 ++++--- internal/processing/fromfederator.go | 41 +- internal/processing/processor.go | 4 + internal/processing/search.go | 114 +--- internal/processing/status/context.go | 4 +- internal/text/link_test.go | 2 +- internal/text/plain_test.go | 2 +- internal/transport/controller.go | 35 +- internal/typeutils/asinterfaces.go | 265 --------- internal/typeutils/astointernal.go | 83 +-- internal/typeutils/astointernal_test.go | 9 +- internal/typeutils/converter.go | 22 +- internal/typeutils/converter_test.go | 3 +- internal/typeutils/internaltoas.go | 145 ++++- internal/typeutils/internaltoas_test.go | 2 +- testrig/db.go | 23 +- testrig/testmodels.go | 89 ++- testrig/transportcontroller.go | 5 +- 68 files changed, 2946 insertions(+), 1393 deletions(-) create mode 100644 docs/assets/diagrams/conversation_thread.drawio create mode 100644 docs/assets/diagrams/conversation_thread.png create mode 100644 docs/federation/behaviors/conversation_threads.md create mode 100644 docs/federation/glossary.md create mode 100644 docs/federation/principles.md create mode 100644 docs/federation/security.md rename internal/{typeutils/asextractionutil.go => ap/extract.go} (74%) create mode 100644 internal/ap/interfaces.go create mode 100644 internal/api/s2s/user/repliesget.go create mode 100644 internal/api/s2s/user/repliesget_test.go create mode 100644 internal/federation/dereferencing/account.go create mode 100644 internal/federation/dereferencing/announce.go create mode 100644 internal/federation/dereferencing/blocked.go create mode 100644 internal/federation/dereferencing/collectionpage.go create mode 100644 internal/federation/dereferencing/dereferencer.go create mode 100644 internal/federation/dereferencing/handshake.go create mode 100644 internal/federation/dereferencing/instance.go create mode 100644 internal/federation/dereferencing/status.go create mode 100644 internal/federation/dereferencing/thread.go delete mode 100644 internal/typeutils/asinterfaces.go diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index e72f19f02..faf9f181e 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -1562,6 +1562,64 @@ definitions: type: string x-go-name: Visibility x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model + swaggerStatusRepliesCollection: + properties: + '@context': + description: ActivityStreams context. + example: https://www.w3.org/ns/activitystreams + type: string + x-go-name: Context + first: + $ref: '#/definitions/swaggerStatusRepliesCollectionPage' + id: + description: ActivityStreams ID. + example: https://example.org/users/some_user/statuses/106717595988259568/replies + type: string + x-go-name: ID + type: + description: ActivityStreams type. + example: Collection + type: string + x-go-name: Type + title: SwaggerStatusRepliesCollection represents a response to GET /users/{username}/statuses/{status}/replies. + type: object + x-go-name: SwaggerStatusRepliesCollection + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/s2s/user + swaggerStatusRepliesCollectionPage: + properties: + id: + description: ActivityStreams ID. + example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true + type: string + x-go-name: ID + items: + description: Items on this page. + example: + - https://example.org/users/some_other_user/statuses/086417595981111564 + - https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R + items: + type: string + type: array + x-go-name: Items + next: + description: Link to the next page. + example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true + type: string + x-go-name: Next + partOf: + description: Collection this page belongs to. + example: https://example.org/users/some_user/statuses/106717595988259568/replies + type: string + x-go-name: PartOf + type: + description: ActivityStreams type. + example: CollectionPage + type: string + x-go-name: Type + title: SwaggerStatusRepliesCollectionPage represents one page of a collection. + type: object + x-go-name: SwaggerStatusRepliesCollectionPage + x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/s2s/user tag: properties: name: @@ -1621,7 +1679,7 @@ info: name: AGPL3 url: https://www.gnu.org/licenses/agpl-3.0.en.html title: GoToSocial - version: 0.1.0-SNAPSHOT + version: 0.1.0-SNAPSHOT-dereference_remote_replies paths: /api/v1/accounts: post: @@ -2395,11 +2453,10 @@ paths: - blocks /api/v1/instance: get: - description: "This is mostly provided for Mastodon application compatibility, - since many apps that work with Mastodon use `/api/v1/instance` to inform their - connection parameters. \n\nHowever, it can also be used by other instances - for gathering instance information and representing instances in some UI or - other." + description: |- + This is mostly provided for Mastodon application compatibility, since many apps that work with Mastodon use `/api/v1/instance` to inform their connection parameters. + + However, it can also be used by other instances for gathering instance information and representing instances in some UI or other. operationId: instanceGet produces: - application/json @@ -3306,6 +3363,56 @@ paths: summary: See public statuses/posts that your instance is aware of. tags: - timelines + /users/{username}/statuses/{status}/replies: + get: + description: |- + Note that the response will be a Collection with a page as `first`, as shown below, if `page` is `false`. + + If `page` is `true`, then the response will be a single `CollectionPage` without the wrapping `Collection`. + + HTTP signature is required on the request. + operationId: s2sRepliesGet + parameters: + - description: Username of the account. + in: path + name: username + required: true + type: string + - description: ID of the status. + in: path + name: status + required: true + type: string + - default: false + description: Return response as a CollectionPage. + in: query + name: page + type: boolean + - default: false + description: Return replies only from accounts other than the status owner. + in: query + name: only_other_accounts + type: boolean + - description: Minimum ID of the next status, used for paging. + in: query + name: min_id + type: string + produces: + - application/activity+json + responses: + "200": + description: "" + schema: + $ref: '#/definitions/swaggerStatusRepliesCollection' + "401": + description: unauthorized + "403": + description: forbidden + "404": + description: not found + summary: Get the replies collection for a status. + tags: + - s2s/federation schemes: - https - http diff --git a/docs/assets/diagrams/conversation_thread.drawio b/docs/assets/diagrams/conversation_thread.drawio new file mode 100644 index 000000000..99d365dcb --- /dev/null +++ b/docs/assets/diagrams/conversation_thread.drawio @@ -0,0 +1 @@ +7VrBcpswEP0ajs0AMjY+xkncXtqmTaZNTxnFKEYTQIwQsZ2vrwQSBok0rmObmGTG40ELEtq3T29Xsi1wFi8/U5iGX0mAIsu1g6UFzi3XHfku/xaGlTI4pWFOcVCaaoYr/ISk0ZbWHAcoazzICIkYTpvGGUkSNGMNG6SULJqP3ZOo+dYUzpFhuJrByLT+xgELS6vv2Wv7F4TnoXqzY8s7MVQPS0MWwoAsaiZwYYEzSggrr+LlGYoEdgqXst/0mbvVxChK2CYdTp/8m+9T+/pXPP0Bw2sCycPtp0E5yiOMcumwNbAzEiOSIG6/DnEmhhBfUDyH6Er4hmCCk/tcDJ+SjJ1ID9lKwUZJngRIvNmxwGQRYoauUjgTdxecJ9wWsjiStzNGyUMFLwdmImeFKEPLZ911KhA5+RCfMysmJzsAhbsknuv5ZXuxDqOjnglrIRxKG5TMmVdDr8HlFxLf/8B6aGINpo3P61BU/LJ5I4BZWHXcAZq+BqYCqTMwRyaYOnooCU6FAvBWIuisoVKDDi0xu6ld/xEonniydb6UoBaNlWok3I2beqPWSzTX3YqW6ldOEwWG7Gih4K6QnM7Qy4RikM4Re2mRm6Gtxc5rCZ2yURRBhh+b022Lp3zDJcHckYo5LmhSZwA0SpRuyl51/dIH0jgIfG2gEgdjoIJeldvbM843GHdJ0SMmeSZVsG0F8xXHmlyDEZ4n/HrGw40oN4h1iXnCOZU3YhwEovuEogw/wbtiKMGcVHhW+OpNLO9cjJUzkpUpc0frvMq7aqGPzYU+biGLu691Pj5i0fSGby0Fqei+a9kcbCib4y5lczDSZNPZUjY9rQ4COrn2LJuOY1DumxDF/mimHqo2zRwdUjMdt0ei6fhdl5oO6BGcYNA5nC17zlfkoC3zyTa5q4McVC7lzpKQpyUhnRQbJyH7hWy27yTkfdQ9m3MOvKnCZ7Crwmd8YM6Z5z0/URqtelHy6OdBrplV2jiyv5LHPA/qLdjA6xpsc1M+IbyY51opq3rXvlsVUhAThm5NrT22EAAtf1WHWvUqanjIGKiBj7Io1c/mnTY4D1qUuuYu9XjhBKPO4WzZgb67emvT4/nar4FdnM9rJ75b1/iuttM8dI3vtmzTPzj3T73rjHP6dnBrzukb1ENzzjzL6E3Zqf9wB1p+8z1o2emam/jegl2p8u7B5s31n07KhbD+5w64+As= \ No newline at end of file diff --git a/docs/assets/diagrams/conversation_thread.png b/docs/assets/diagrams/conversation_thread.png new file mode 100644 index 0000000000000000000000000000000000000000..b4eca90f7f8f3cb719b1ea6add13f742c52a58c6 GIT binary patch literal 23930 zcmZ_0WmuG3*gr}PFo3|&N)FvAAt}w!(hbsGQUe0g-3kauhe3BYNP{90(umRmB8_;~ z?Dzej>zwOc=fnQ6ad>7uEAIPOi?}D6$^>{+cxY&71S$vx9W*oyV>C2$9w;{WiA0Gn z0U8<{O+`Ug4{33jhu^7TcRft6Ncqa8WSlvEgq`!V&&wz2jH+d1lkDy6E4Hc#;w0X7 z+!ghQyW;UwFJ~i32v|6DIcD7KXRw@7Zy&Gu+vok09$wr&=$c-2sqsHtJH7hzdGUGG zMeyoc!?{V%S?6@w%v_aeOO65*8n(c|Z$EM0v=doZW!^DoKlNsJPBiG!lUl^}_02zG zCZ*P%vw#rK%fCl2JWhX3HhgPxF&$2(J*;e)ieC|ahlLv^r(BqhFetLbFnM7HyS&;i zaNFpQ^1IxK>0qfcYnySNeP`9={55`@#HJ;=rswY4&*=h>=LdB$;D?-k`(MH%Wr~>K z>H0f{p>y>35b4WxOlk?gN@3T9Pt|QdsVa2JRJrH$9TSWi?30qHgvWS04g@PKdqRqM zEceMV>f~jZx%^nahT{>3bqC+H#|WB+lJpRV+IHRi>JYITBdaiLt6xhLSt>oU-vaFBN=4;Ou7QkD!smcJTZA~zyyg5g~g&{Fd7T+7iT_ch5p(7 zG8}k*;FKrf&ua#=ZvXwY!f@$ev4MbE4Se#L6g`dm@gZwfRH%HRatfW_zrPozEA75f zhzZ!`&s-1J-ca=O%kL?}u68ELFm@U+9=zc>6Q>E-LcLuA0j>CK*{#}UTyGp(z9V1f zak%~mJ&t_**$at zaQFQBP7Y26xjT+|Y;?5RC7-SK^1pwLlnDgyTNaa*Oo5Rm7ivU%hD>>-QvYG%tD&V^ z%W9yD!SL);@TkZb@D!wx3oApaFBoY(p;IaLt6(NOLJlU#j1pKJw-KLkSS%sNcK-9~ zeJ{2+HZV^ojTHv^k5KcOT-Brl#v&;%vlx~CKz$D{j0F66Hyy0R{|+-!h7blD9YcRd zr2~zO3a!T;PJr|KGCq)dl>PHafuwyieu~Xy)i1V$uWu;;OXR4vpEQ; zEj@Z#<$(=V8L{0OyRm#bH0%$UCsq1~c?=N72=G8AoA@{EjOiJqM#Ar4Py+DjPla;| zsP7N^1AF3U6^Pt*WPk~Sw=1?M-u-X2{~O*Q2KcDaYwZMfJ;*C_qj#C}?NMNP(ZTZ4 zS2EK^$&f(E8c2`S`pAbNOIC2+V_KZNCrt=v! zK0BdR@St=|wJm!@TIKTk2kj^sBd}mmQH?)|F|d+*w+mjlmnbFGKmXPC78x+OE>CFc|7)(sUU~WpKZ#)yxxD zT(c9;Co1IV_=LlyqRfy=gL*|;xn6k)oDC97D)WnaH2l7W?^k_PO`r~2iL3YAkW((M z?)*LGRw)RonR2zJ_-?87e~+;@T<;(+n~x%EzWZ~Nc`=mS+JdyxIIs-!o&WbE;K!DK z_YA8MZ$V!{RhZ(GQ0w|)4<;ve$ic+J=|*$^m9Hb!YGh};vqKCJRMF%G#o%*Ok`O6| zBJ>utm^Id=WIl3mXc83uzlSE)c4(oM@$-pDwSgfFRXifIn-KKGttTI;&4*57=mOc_ zUrXClUOOfP?Q)ak@ofp$z(xqcH!SvPmq5cR8D;cOM_;lnRT?*#>DsDTF%};5_cwMJ z(9)w0rj1$^vl8@4;%PWOZ!N1v&Y+{dNjwZ3^uk&&Nu1ozU77??RZ8IDyG&36WGcZ= z<1@sX?iUajLbs`6dCVLg7AQ0^-V@M55zS;H`hWkRE;X(jv` zap;3Swfi6C*DEm^bIAYr5msN5W&d*K|12XO#=pzq$25IZ0@jgdhfAV7O|Q&fQ<$yN za;N{u8!fGcu}6nW8n?qEWgsx*QP%}{8++nU3Y-eK=ysG!qf|4B(9#r81Fm21-eX85 zhlJdoC@%$U=QRjsViW&YRl{~rwNWY;QGy{0rzR-KFgJozCS-De70CjF4R8|SIF!I} zc;glQ^_{j&ZuInSwVHs$F5}PnlQG^u;}&h7{J_4iR({WH0(D)iA8K9;xp|gBGRjHG zNDGJ5r}PqtEYrV4eO{1*4@2)!35~e-{5_|c zF5Wne>K$b(a0Rtwdj3z2r~>3&nleY0h}*YTW5a5*-Od$&F7UMRH8Kb>EMNR>ngj>^ zeA!z|;?E~}#rhd^tzo<{S0?^vTL_8cUQ7Um+!yIgz7N5|G;&22zi2(q`$HfWt;V~+ z!eiOJ?A~=|;IdF#L_jCG@ImjrPOZ^przc{5=X1$n{Wx#K_1S($*~rTu4T@zQqR>yr z0W+m4O8Wl7LFXS`b@ZT-;b>OdeBvW)|WGmh>MjQe|B@P;TS; zK~0H6pW#z2m^brIN|RcxHkU}N ze7&GI9{UJm&``+n6MWyK8x(0J*x#+um*!~Y%ydw&Gd{8?9n*V>k$}T^EBh4SH$~I# z-?mpaqE34cFd7`Do6irIw#{7XlSJH?9b1;4sZDLqKsMkqa`I<;Uy~k5s{|3lC6bQA zl#-~1{L`?y1=A`qde9ngm*Fy*PhtpZ4?uNSt=RBp{r05BZm!Z8AgQ*Hh!q=OT40$h1iHlup?zE&iy)guf#4?cWVKnp9}cxWoMtT2!X z>h6^DT;+rF#vHx`sOv&)-v@)LoFQUiJu>2*)vgXMgjtLG3i}7PJkHQw(X}8OOzeBG z(GO_R;`&f##ZXtd8RS4F7CuF*+W-ZRNo(PFv)fX1RrpUeO!Sf21io%Zi1f?uwHnJF zp{cndX{Jz|z@1M;BF?ky1z8L$<1%POW#DKjM9O5sV&gA1l0;WM-S?coepvjGW%sm%XD8Xw|E7t7vmW3M=N^UJ?W$;vQ~lgg`1 z?l2TaPk(NR6Cy#4?*A?(L(~wukV~t1zEsZr(p40E{crTIqC=bUea)d^_h5x?Q&usw znSI&fim0l8YzeT?{eN7t%v^UOb9fDry=r`2Ti3*@|NZKsj7-?kqKrVh4SNTFdvlsS z*eKKbfKQlzvA9lB`R_C2iZaX<#ZaZ2AgpfXCFvPnMks`!KsoWgRq)MuQmlcFJouh9 z;wOGZM-<$qFVZTJONJHWBp8-KCs9&YTQ38gF5i@~INk>n1)c|e0Lo(Yz1DJ4)O-Ec zmhf=UvAQV%@|+POF5EcP)snLN`axgqRGpR=^&9N;;5RZ;(vFm4ZLi#AcMC`%v&Uj= zwa!n#6G>oC)}MFs;n-1rTJ4n)L!Wg5-%uJE>cQ-?J5%O2%96EE zteD{1vf|eu8-cyp8-}@JQV{U-LC3H5eOK^})rf~Dh{L58|1|=7>1EJzY(ZIVU`mmk z`PFq{fvPxm;ufmwO6zbi5xr0y+5a6rDSAOYev9j(HK0S!aAg4G>PD5RcAwqN^l)&p zwnx5@YDrpws@CjK5ImPw2k1w~k(2yKwSBN}f3@3nq6)=IpZdooyDan0HY0`lPJs+E z0!;L8KLB-MOMEXBgIeZZE{OEMTW@M%=T|G<(^86o7r$1P2&Mi!aarwbV={+T$egb} z_sEqFZlBDPn9mV*G1&TQQfD0{;jmb5_aJG$-cHTKQ`fIZv^>4IO-Y8Jz&ok-BZ^jO z{&%uX8XY(Agy@6+d>+a?2R)%lxsJ@dI?rD2k25l-pgI2_OM>qk#%~xX%=iZxJVWw?VuSfF?gCuE+SMJ!exVYyja`zi)P1Y@*&l_ezLp-yp@6@4 zbnnL`YPVrp|B1y*DkIdYrN;JxV&Z5soZMvU(e;&>j9z69;K#!c71f-KadG>mt)4|p z+J+VZKSZJn1nuIP4B7I0bn1=(&Whnu3pWI|r=zBT8p{|{oNxcY?zjI=LFjV)0KUo_Gur+{->e=5l`*^Uj&%tVrpj-lve3l_)efx9`1NKF{!y2 zxtgtohmT$ZYXuu5!EPm>EQn*Q`|tUZ->kOsBIt81SLXl^dX_z~pLkt2=?_S&_x1*s z+AHoJ%Il{ryRUsAsN%TP==73G+JV-XfE|>>&|b^C-;_+O+qDCP-ce$t-%}%X zHsFQ57yWgYhROg0h-)l>r!hZ&Sw95G&)voUp+l39v)ceofEn zS!?^{W&$lo|979=vMh1y=qJm`Hh0(aJ>MuTgRCemkVX4Ngpy!Y02kvnwFRNy{KDs$*psk}8_jz~G>WmJ? z5SfBTcg2+asQ>!>(8cCsqKX4jkB8|<9S5I{W@1&onh2ui@t z8o>~!7LtRu(v;&fI|EMbJ9df_-moPK62^~?&3`Qnhzc!$-rlVJgE^`!#?q7=wP>!m*F0!UPMMH}^lI)8avC z9mlYz`0s3GC)FJc*PoYxSg!jC1W?o? zZ}!90UPX~|k{LC7xYyf9U| z>WSfxx`CYcs~l0qjatvoGI07lT0|nlZj?K0*Uzu8U{`qF%m&q#0R`kI1N`|u-b6s2 z0y9_5vCA_QT#=C^hITYy)@Au13-8R^^JoofEDq!2>V=IAkAhyBQJB6AhRJq%}c?9!ZoOu}=Zk)IdCxHp5(7l*&jG37qWm?C^}HEwAp z@cnd3V)|n;Ew#pRX~qTA^;_d;KEOU{+($gdq4-FkJ{|t~qQ%#n;W8@MOAmRrvS{^t zYn1TbrbRxGV!-I1{3>%>YQ78Q(CAW>$>;XaU>gET4476$_v!V8;#hr(q{ZqTTG2yYz(5ux3x@$LeJd+tD|zb4rcca zac}!UF(2X8aoF4tc(&I@m*kpaYq~?El1Q!5`X#N<(so@RvAyh*Jj6BA==8-}|4!_2 z<<*zZO_eq;tC;dm+PuCz$}4osDbxaD-Uq{){3)0Pjj4%4OnUs=nMHzT_~Ib zp59KE6$bg*%OD=M3}jhsgQv(D6*jRqmni}7y}on@KmY!k)^DMYI0w*psuL+8_P0@A z00wi_!^6>%3~TgOZSDqA8JVdv$|TuL;PbmSe6KUKNS>R)0)y6P)?z)oIc%Z!lG@=C z=s*cqAD>oJiN}PU_^$;!z|`YHY33B%`0(i?*U0MU?`qk1Q0V4FYfxjU;_Spdtr6@x<;%;FD3 z>49K?fCF2aGSodJGXOO1sWNFA^INo|w8?#)OKw_P86&28YFjrIg-`hn9fxK;mT0L0 zfQy>Lx?#HW-{X88oa>_>!q$A}&8NzUnqiG7K7ys(b9cR;Y^L+K!@R>!1oY^Oe`J+! z1Z<*%!4^(_T~9ZLUTe_FM#`#=XzvD0rfxyJH!@H|w_M$8Fwr9SOSy)(0*`*!eQW9i zy`A4RFaC~q@ypb*@2{75(#m~gI7L=vpg(ZqdiEmlWQcO*?^^)p%I*ji>xOw*j+g z)2-n2IFvz4>P(M#G(1(vWo{ymK8Ogte_VkZw#o~Y<7Zo~pFvP4+?xqsmwVg7oL^=e zm;&6YwBgSs${i%MzI(GAg2Bej35UAQ^bE_#sJ<;qWLph1e)rCq4`JStXBlt}^tM4Y z<>j^YzK9hhP!u~&MO|>Al&fr=p{Zgft+$($J++VTPEx89(e^c(rJ?0sJHU_{uVX9wo0pPy%n^nCu}xfP}-R6{6sCCwagni;TiEGDGulH-^ENm%U^d& za9h`GS&S*kQ}u0PJV*-8l6k7Z)DZPh?qgew{#S_lh4g-#!8r>!vZ}RC*Q&s6H1t-G ztrXyvp-kFUcIM;g&mWnyR{`~Ux6~X#CJSK*n;5FJd(j(cfkq3`00mEgs zCiV|~wBhHY9+)0feidMD)kFU7@?csM2>ifWM@dyjwzNDDIUu!iF&7GJ>_K7QXMhzk zs};MORyJf?mp8Z_rW`Bd*o;dIC8gtQo!xHh?_#U?uDf@C$#3_^L2g-P=#Zq za!B$F+7dTTQA_31$C47AqWq+QykmCa-oF6Nws||U$+41}CJb5-lMc$~l0CCdJN^V? zCgfl&$0wr7C;b?${+Tm8hK8B2UvU}X#EhS=@I)k^oh}yA(;$_l?t!7W_;GOjW8KaW z1n~ru3f^PXew~42i6vi(k@aJMVDjOR0O)C?p-$;>>pD=Mr{sz= z^!xbP8IUc80@4&?TkvP1*%A)qpeulcta!mNep)Z3@;n+b0!WZxvzNl7mgl}B(Ulc- z@^{?M7__=FrBA_vU8C%g3mZR*Utxsp4@$!5u}}KG_Gx(0&6G;Po=<40aNyrC%@Q48&5 zd%!qFBINTkCqfTMYkW`$MVrR|$pQfJ$@=~MeU-isGi^bOgxTXv+A{ei13Oi$`3G(ol!cBq$!00n#QoE<|FlHUnw>i0< zIKvoV_>gx*B0vCAu6_4{g&4z20i5(?gQSI=@aXBzCY^mHMwDIvw5>uI7zihXYuhWS zbVk?dO*3cI>akdg{ij(u_z}<>^WMAGOc4f%#*jgA!Tw7P<=t{pN~Up=k0Io-g*N&z zJoyCuWJzjW7CnU({D5q0Q?VXz=D?)!KOScWnf)YY)+uZ%gnHX(TI;u11ObA5l< zFf_)XoCoRhxi@z3TD$SgSJy@Hw!4C3;uu0fKN(})28 z)}H5zdgOn7uRAR6v-9*eMD67(MCNNBzYw8CIu?D_yu*NTsuac&$t~Px8IcJxn;Kc! z?-iAnih%|gT@h=q_5hvp4GLSmqoqV_MhN`~WAs^nad%3OlDVkF-#Wc4cyYJL?h86O zwg1nt46Z!?22iY>l1m2$(v@5J4V4MFmYt|GX4@&w#ee*k`le8>V=%X3eao zE-nl&@zLW{7rz(mbXiG4Uc)r^Fv?SKPiXWpfPyDd)%fZ`% z5YA>{y^Sz$p-?Ju2OOh49G%pRwfaF12=Gzt)UtS+9^Y;deUxg8wM7ZHW%KbWH*V{~ zE^`aOQ%VrnHLKa#h@o$$c5m%FV{!$=qx$WaOI#={f?RK=K*TKvYH+WbFxCJ#AU?yyqG47=3xt*ZnF_3;!%>9$%(<7l_S8tb{)^; z@m3E6(^7Mcv?DkM)@&>1FTSon>EC{Ck?qdl9DS|G!hhzFkb20fA_;rikyRxh?)2}^ z+sBH|+OX%%SOIETK+~+{!2r92_uFqCMq}%llXg8+Tt@aCF z2soL;ci}Pv4w8n0*4nO5(&-^U!QHmDYSe#Cf9oyfQfr5pTg))D$ll)-3Za96Wr`VR zWRW)`;e&Kv{iLSme;TSUe$;u|9IrAw-rDDk!8*C5(;OK+4XXqSGDj}IX`%fi%~buJ z=jLVivgmKV{B~OdY|HjYwC76W;Dqpb*sNITTdqaNbR?t9h-Mk^Bz(EfdwIYUD+*0|8} ziQe{?HfRr&1J))2{4V-TGk)=(rwnHN@NKUmt8ZD?frggBO(znDUFFA#n%Q}PPsNB9 z+2hfID*1`w!b8p^=#z=l-t7B34tkrUfgtGc{f{HOJlr>$*@8pFl_*t!eG-mK6sDTN z`JgL|= z?zmMC8z~ER+DY72O7QNt|oq2661E(a~ z;C@%eP#-D7d!R-?$k%D0r)j8VYapnl2etEp>&(C z@Pdel4}vr@RQ(tT3Ou^6?CsjN#t{#uOy}h9r|5$o;098A)y@~dzOqL)!Kn&DuHUa9 zzb7gEeb^l3^!^O+?snEeWQX5^6;4*#*p;jctK^#{x0XDV!tLI=fo)oE8#k_5QqucI z%-akXO>hoKjD9!A14M_Q@ic+WFZ&* z5#(YBB_>c--4fupd0B^_9)b%-_Msi0~(kfyFNJVx>#L$XQ`c@ zv0+#CMNR?u4GiIVG4B=Xf1?F1=w@D&mqt%pnKC~=%{YUxI)!c40af11)(+4Ql=Axx zz;eewWj5mn<+<+lK&~(WovJF}^k;SPV_QrN--{LhrBaobW>}t3YSU!0(lV+?3V1rb zZLms|$e&hIrk3>$XkeE4iDD;-L*)( zir#>UN@^p(OCN&RrRU&2z*k(H_8mYRi5THmLbVrws1&o^VfkRda{({m#;AxM0Z6PQ z&38TbmjL@EkBw^)8RumfhhRIkl_fOx*t|?_?R^>5f}4&*Z2LF0D?J@iuOpU!I$veF z9pVjjVaTP_vQ|-vv6VeOKDN@WMFlfq1Sq|kKw?Wh!KVS=yF3wt1-M_dx zpSqHwY@1|uX6$mD7o>a{u9BUaWh$Hk#(kH_ZJ^I~lJ6HCpkLOWQKfc2%~wdz7&~!I zD}EeFVwIc0BQKdk&S30%^)MMAq`wZCU(hqmU+W^71X>XW%oDI~5?3CW7Sjo3SUrNm zq7VoJ-v)>tA9SjavS~H~+jo%{py_Q{Jm@`Bf20Iylndq&#gZ>r>lCTKf4(ScL@-dL z;>K!oGUJCURek{~Y%_X}p9M1cxSVlfd~O-o^6BZ7d4AtGc9Emp*&L&)G%2k-lKvqy_1_Ht)p@G9{iE28B%f~t7P*BA zFrw%)R@e7k#+}^P+?~A0F1YKUS(#k@g1Z9F2#skhvI6;>9%K|x8$7j#&Tcl z?z$m7;t|Mq%{awC7@@D80JC>!6EEmuNQeAWgRNf?m9!z+L~Nc3#ptFi;e&pB{hZ@a zHW5x(CkN8#Scf=rYL3Cwp9aSmzaD?ImI*)2s74P#5y?D)9XUaqpb&H){XA|l3NlXb zJt2^MsqM`8U^JZnzjwlyL%4BRPsR@SpMD%*)Fx3e;2b6$z}AlJm2+gshFPk*!oFm% zj~*mw1oVbw0dx)Gu3L7fCF9l2-Y4f1zCf#w*hl3dNcl9Ghb6)GNKQ<9FmZ8Pt>R;) zR*;JIiyrCm@Bv?}!=Jw27U`Ah*&YlJcL%Z82i@1rpV%1DhZNiG8(BORf&e%k;yH+x zLSAYQ!tKfG0IN?7xu_H;$P@>k(0q99=;Njj%}Gld?_y$~?9T}}fj`flutHD{a>DW- zB3&1$wnIpfO$L=l8&eDeA}>#3bN?uIj`Nbe)ciN|dy+Qq4iEn#@i&5Eq?SefoT^M5 zzT~>7?T*f;KWbFjuPB(qMRcGx$7Wo}O#4nsQ^-v})?_1_RpNC$KZ{y)>O#LmB;Bh& zl67vehFm)%u%{rnsy)0~(d#d;`-hM1wNSSCL;DF&0nsS9Yk>An+KUMKM_4SRkBO)( zNlkmV`~1m1D&|hcuOt4x`UuaN&tE?gkF7akN4$J$=wr4~%Y#VUGC(DcJrfzjtSF8u#N*WH?{M zbO3ZCyu6oLIFIBp+pl+kqw&5LYZdU6JW&hMe2U!McGWl)t3Do_9zEDR75CjE3!VzR z;=bhT4a>zVC?fiYZJU&5%H$Qy!p*vEG}gy zOhT`ool^O*Bs?@lf&e+V_=`!pVT(fK!jD7&x%%o)JIUK2sq91fUwPmKE%aQCGU+TF zDDB6B@RL+t_Q_I{@6o1l(nEed!yKL*$CMT*%$imy+xE^;V58FtW|nq%A$8*I43)iy zP|>=TG%aOHD}ALX9(4KHPFER(Hi{o~)sY;IA8{RrD{1LEz2d--PUjYXoGLS<=02K= zX)Vp5-KS>H$ZPvk?tcQqo*+q7{yHWgKP&7A?G?u=y0oEsJAd zM9Sm{-ovNh_T<}OiTX`o`<-LW9ThKd?r{%8Aq9ob8}pG@{ z)ML&LjyoM-N||C704maJXeV%Msv-DiE2?<&%w-Q^xp+8Uac)RSeu>i%5dpbP3upw zRqF&V&&6cHoO^zFNqG_|m z-0y9q+vW^{hy6 zcTf~(&sKYz8F{I6G4sm)2g4J9qWz^!hh#lu#~#Uqhd1*>DoI*Z(KSLSOHI7pZb4F| z83lIpo&(6d2m%;pEDjJ1C3TuXF70Dh;-ldgNF#Ow07mMnGo_y=Hb1X<}&?HA`dU>)$a?cVVPMuhf|J z-=}krP?GQTvA*+6>K-hSQBtXdXQQt$e1OSb3;&i{`h%?LjY@xDn zh$Nq^c{##O#N@~NTy*D?ss|)}{>ZLlv=hF_Xe8^1>F7OC#9T#9_nzSTyI^mWsR=Gv zY==gT2+;4Yj{XE@bk%7NR(8#wiwH6~9XJS5Og^K7M`mG4s+n~FXU>DTsu(6|iF_4H zG>Sdi??N^)eGN~$C8)z3hF?;K}$%o{0kRQTv_Al~gut7E|LN}g!2!NpFfPwn6pA(oE<{+%} zW|>b??O~`IP|{Z9U(P+$6G!p=<@5U%_|0wMF)o?q%l5B9Un%2pq?@34V5*x+C@HwW zCt&pE!!f}B1*X%uY;V?^@i6;H70)^PJ_yCIK7Us#Qy+Wt{eI^oq~;ZM-|3*QTwJ{E zso&A+N3qy3nQOm@sCM+)gtI0=W6kQ&2de-Q+Q4Xid9lK*&O4GJg$fT9IlxLd4?mOtBu7NN)*_-6l?Sw9n(};{~C3A3DuBYUrsq!LbKx z`ep9e)~()jT)&6tCw*ov{feuAH!WdDA^myuRchzF>W2942ub$EWEtcW7;=2Fa|=`S z%TuT3=WymWR+ioBuFCHKxbHc4%g~I#tU)>;q~oMEf}6yZmMnSVXJ?#EhI0J^@At46 zf2{O=KL&+JH(q`|j<_x6%VZ-)DCSZm@}sKSq-K<@EFyg}VLSet{h+QkHpHAkc#+i@ z#_WpuS>ZT!!U!hK@9A%!ogJQ#Yi)SEy7ebjJ<7w|)TlzbHL zXNf;eDQ{M|`GWY*D2x;qJ`04RiovD_XV$gyMhkW2^zcEy**8{qn*`gT$>Op~whU1g zYo*wTMId%GKPX}z-9)3Kfk>Q&ZZN3d5_lYRj*@Z>ycne77hPf*JU^C+cvYK03SBZn zu|JR1Fg7k@eu3cgSUyHg9F^N#9D1S`*5=2ppF>K$x2J~YbPUiQP6o!26OV+^1Cv-JBFxzBhTq0c+l_*;Q%lZHYJ!D;P;+5v(zdkK?qd|** zZ0inH%*cR=CRQELEH=($ysoZ;CcJ&Db0R1+L0OtMc4Nke>}Z_}c(6gWk%Na2*{1RA zMR(tSRizLjO15K1oxY}BZ{hl5v86L1VU0R6#i-5|Z(gE@jb9RE_!IX0S<<lM-vR74y)^U<)FgL{KvWx09^S1WQY*8={48+U0 z2=Ik)9~FuF-ILb=e)#rEeW0)s!lUvYXX{GH~c`6ktfn-3-1zOr7v3v99%UoXk&L^RK^Ta)*#}50n z{U7H9dnBrvpn*>5@xkNm&9N>Km$g4#EfSVVkz&>w%Y5+Hollv;On>MLtJFRi^T^90 zw5WuirK&eCTvcsY4Ni@Q=z85A{@lwi^eTw~qGE}|%Sva+OJ3uw(nj1?tKSBeg7>?b zNBalR;x@z7uOCH5S5^ap*KQTSzsUm+!71v(5I>&NbR`lsb+4Cvip-5dg9sQy2-)yi z%QqYa(}yAm1R=)klhiSE%kR|ipW4$7a@GAMZ4B$h3^h#T;ijXL{Bu&xTt1d{E>n1m z2Dz7l8!0@$R6rgKeH&OLJA!;v-z-O3O+BKoGDC+=xLC`_2ZiOs<%OsNy|sE@PV@}U zXz!2qg!9iJ%vjR0sT>ma9gc8-i=rx|i0)(iKVR4x@G z9ZOb~LVd#2OE5hu3)_QAA+Az|E-^8eJ;=ZW?V;@I;Hz4@$QXh?rZDBi1-PU~DtEI4 zgOzKAmzfh0gwB4paI=$Nu(SRI#(RVj@w7H-q(`WmR|vNR;UuW;ZDrZ%dHWIMi<$AK zc}mr|rbO6Sie;wQ>GUr342k3AQ#9G&k2%Ma#gcZ!i_UX}&On z?>^&diY;Igrb!lH+d-(4iSPl(prH)Yc<^*^4lA58S%-2k91@-tA44~+OHtH_gFz=9 z|Ah6l+-_;T4GCr2L-Yc~`*bSPib3@af?}y2a(9|*WBzJ;M1e$wV*7iYwd9y1tYO%r zAW?^eRW(6$;LCWOZFTzJF*O1micwCUtKRLkYJ>r+Vg8G4_T!buxn)&etcy{|yx20v z{O!UBLe5NcBOPFv<73F3C@tR%3T#|4+?U#!SE>~@^%``*isXPBr9HhlfT4#=Y5u3rd$~Dr-Bog`Pp8M>wW7?p& z@o0Eln-vcFLMM;z0*MECogmWm^~+qb!Pu&S!E33$BynT3hlBCGNAQkRZvJ}mQFi8h z0zS&%Bj}W!^v`I-=5BG`y=-0R5>q8t?4jzCkyCzZMT}>T^Ug61L#o=1&L?5SGpq48 z>Wg}1Ao`K;ApfaFAl#UFNTwB$x$Zye@#augFl=<0Wg#yM-aPl7BgQ{heIUK(ee0c{ z?vTmwRT8+C`?uLr^_zog{e2n|`JYBK%mwHZZT);%L>p$im0r5aaPAwD9)u2_MFzVn z&5I80L1XbB*x+KvHun5TVve()RRG?p!Ps0*P5Sggj+qt3octrhYnSDEV(56c_s=y%e?ipr1Zg)7V7nigMM~%r?$_E%v_?aAKmqu2{`0b4@ifwf?qybeK!yYy|LeK z-Jxy{UcA;cXy7V%{SozB#%X!cLEjX--$sK0>kgsp? zt+hhHZ;4Z3u4CFUHZdBBX)#HwJg%J@R>upP#@S<-&Q|fW8sT(qzehZ>E&oV9!`Flg zx>c>G{1Xk0mIL*70mi#K5p}_cI;&1Z+=E7^cQx|5GH7oizFBe5RLe`sufNjOtW&l3 zAzR$b-qR7)hdetgSD1q-N)C*bs`++B`Ost=**<<&!s@LC3@di`am?4PV>49V+ks8C zUv+F>)w7@+pc(nlEb{tYE})>=nGThH!JK|uvWu%+)9i|G_n`HJw1oVX^~gC5RgEj8 zmdiu+PxRnEJdP4$_C?7mjc@5C?Y4aEz&(r&+|;+a5vV^bFiQ-s%Ip~h_}liDa(z=` zTI>(_Qq~{vx~Zu1p&TK3xnRgKTB$Pa*#Ps2%sRMp=R5h1IF|6R_&0|T>D#C5#vfg6 zRO=(qCi{0cRILnDBvF^!-lCIMB50ue|6=)rwJoedn@*HU5W(f?y(JcHy&BsLcvAWUmb8!{0F8N)% z4P~1gzBwWKp4*5iJBb6uCn zD*1z&`&GxsH9P8qPtzEvm>JAH`>+%VX1T=uc&E4hLqOh9d?ju^U5_Sg&^97IrJVS8 z7+RO#Zb`zUbQab;qK8}VQ=uv^(GjF`{88nf;=e*kPyQ8f9Xnzbr23+6H;pu@j1q}| zev^T-m(HSQmZ%GtM%I#i4g%(~`S)#?>NeIK`14Zor20jXE&uVO7QmwDoGg(aAUi2``@8-Iv>q~ zRH6MAxMSoY_58dMSi4zT^%UGj>&*lz^FPozSZNQ}LV&?<$yG=|pF0fqMa;Z+9fXn-FvsnHE1fs3h@vth81yEEK1%DAL%u^F1VU^s>ZIdg zqVK&Ru24+V7k#o5HhIR)tYH`m?`>X1N z5eH)@1AZyq&RleaRCefV+_2hD-TUaj8sOkV z_L3EK>m(jO!i>PE_dXb=0G)zt* z^chjdLA@FSuenR|1CU%2c<4xz>5URkfYJ0LKJ&w@JPr2;VQARIe>b*W0j+SyuLD_! zY=Yv3NEtfRDluK=h>KwmNnaMmXY64BYpeI&TVbzF1?5JNF=tVT3^fF9cYs}Wzg}jQ zeA5Pp`jX36aBIyXCF2DB zgkJ{|;TZo{A7>s9_4@X4=1iHxiLnfo(#*)dREkPS_9a3nYiPyb$v8x9BZqC7sYyKZvt4l)V&44$bS!~TD z5q$v-TTd90f)~JX;}4g3#iLMjDa?FnJdFtVhq?5F63>qMu_tgh!_*-Ru)yX=B_iQ9 z4})(5iumZgeTkP;jE!vAmRkS(!$mqiG~Tu;`(nJ&r7C~`N5;LK&W*%JH+X=NV-{$p z{x8%GKpvs%HLY%1?IRfMhvq%1y8u^yvIh_dC%JyfDVP}bf`VnTl)p2q;lddpvxo{f z4y4^rEJpca+)7gJ+~zV%E}E^0u{_Ihs_ zT=6#mc&AeHVTNMpuB^k~gkbR1 z&Wby#Em>Wmq}3GkdWfPtWvvX7tEq+;NLL9|h|sSSZ4@ z0bKPNs=a~|Q<+)Pf%43Mu;avQ>Y6pUD04LZe>4<|*2hqITC`8ENH7>{j|VCZJ(W!~ zQAAyRSr8X8P;iTR_xeS;AeVJJn^=NSFVwyHlbh2^v2pcQ%K#H8GV!=f#;=h z69x^d2M5&wa66q{yJ=~P=VRZav*x3%0}E(t>S zoS>qDKV!#u42iZ<|BtrUqoT7R-6SIvo$H71C@Rt!II{dT;X9AzeAOD+`UyOl}Q=Fb~XMM9;nYEALW-5lEzS1`N2anMa^1s;AW$@GHE~t1D2>l zIP@I1ImesQK5YIoPje_wULT9|OTQsnJ;h5i6Wp;;8f&FDke=XtdKF!`nQYO=LGdC= zW0Z(t&+JwcrPkoHu+ADc%)_cW8Dpwk1dwF4RK*=YJp%yR>;*eQ+-HuMqc0ut2l3~3 zL5-v$VJIWY4~JC64G!Mv#`t9NACQd~K&)VBG~6|IdR#H9?x$?^B| z0w1}OE!H==1>p`r!*vo!yfQGN^wfzRp;BfBPmx7>mXc8ItLy}T^&4)#s|ZPs_5vXLccaU3y7V;vcQ%y!8C#?3HwD5csBo>~iW4?Fs{ z_K+V7rtGV-(neqtKu4;#nJPkV#pQa z4{@Vi*F{*6DLZtJUqJY?StIx;yb+y%7Rm1~bhBV1Qzos7`GR;8)Jc4@=j(@5Cp$)q z)?%)#G+%+($YF!)5|bK^Cm(;g@%kAb1!xv!5v~Jkx7{RMX_8-a<(2b=7lL&F%9rOg z`CY(=Q;r1%F8DscybUaj@%xRyec~BjiT6fWPlMu1eJ>Rf9)pq6v_l*P@+$TJzWhDq zt!OB(iVgep2$>vEzs`MEP3vuWbA-{>n-KSc2LmwX!M0zBn`XSt zNdpR)bRRLRec$1bCg;`LpJ^$X(C1(uA-wr&t;kSr6bjqiZ76k#axx;dAdarPbn(T> zpkWQ$D?Y^tYG$`UoT#$ZC3GIZBiB+$&|Ox`kPerITR-A~Q<{w)ejbSHH@CM7(EZA| z$t-LZ2|FDIO!CJsua2CR$bu$d>-PsMpy-mwXB$rIz!h>3uiL7o@s5n^GhPT3%_qv< z3Wxp>tw@NUx=jlSp^A<+_&ihBZYOGhuE5)4*FavXANKs&v39A4=Tt3sptIeMNpc?f z64xyg6q}H==lCN%gwYy0rH8>O|2Caa>DMpA;>&W^rwVW8v(ljo_sbe}K~!q=(op5{ z(uD+Lql-38Lj1L^C~`3inJx8{c-aWL1oq`+Ido6th#t5x3`P4rKQB-?c*cKwE)^5M z>5XcRAr7Hgwi)Id_rSBQ(Qcp3%MH#qqLM@DbJ&~;RK#jTN&WJrf++dcsJI@p4*UJ! zhLVRtS-cmoN3j34geeKo%WWV=7Cu)0xy{qCHO0sQ8zt>zwj+ zjcKDp5u&$Y`l{v=+gv0vpJeK%8eoOJn2$B9b7$Rs30qt_zEKG?5}jM{B3oSebsbyTq*|yPTM6N2B3}1Dulv$$az2TeNGqYant@f|s(Ju?V-*ay%p)7* z_dy-hZ}Z?93rf^_(JjtYsAfx`ajb50B{DftO~tJiH&>?BBEZiXnZ9vAE(AHc081o{ zG#ghOTgP46>KypRMy&bW?bi3W>p}l8fDxF;0Oq|qNfy#Yeh!yu<(adaj2;Xx1m8x& zi11t2PGw`wjtf(yd3|U>OAGE7W?t;PI;6>5&V;N~3g)>wsjrcjYeWd|JB#tr#@`thTiIBZt{zE(@l8Ub8Om!8+cB)Obz1(&! zKH0cUy5l_{xGh)f0O5NR%J}oxU7kDJzVHQ ze?6ou7k6?%pZlb(@qG`)lA24-aRg9QW-c>j=z1n#EAqa>DA~*M^ z1LODkaFnf}IM$;#5Ihp=PGvz-K;+F^_Z5eP_1Dp#Q+qA0>>vJ&vVFwy_+&$%EkLr! zTOyaI97`h=f(O87p}slI-i%HLa|A|})6%S5U_|n6c9-(-@&E+gDgP~rJB1yG&f7Y{ zNFp$@W4!}B*&B$-{fymQs&2aB6#eH!tBi|S=#N@vV$CMkI=*L{w|s(?@XJI`hdLYv zw0zZT6tI~$%#=t}MX(iWt$xzbA;h+e2eV*EjsXF~Pv06IjHaQPn!0&X#+(&tqht8D z;p=-P3K*WMNcdH^=mrZUXELW#AG>m9g0@lY7c~v*-!_3z!P|@ke^}(qMVQrE7wt`Y z@2|h7i5&lvTDHjvUyFG-c>jiI-*`d!P&m>jr9^P`7)uowrQ{a)0R=y>jPy78t4Vkv z46PWsqf%E}HMw0RWhHz5JZMo+PEchaG2d4(nPF0TG$Zb=b@{7U+fREjU%!7n`E=SA zoSPClu=2g*qNB`W$$_j)S>MtX@eV5%oL5~Q3UZSw2HKTxBks=(=^Yb0?g(zCN4S9`%99@w(Lk_x7zZ_lv=w$>3~{XFUu<@d-P z{lM8}xw)Bd`?1`#pvrZ+qjgY9JKr=mfun&%CJND6C>8iOaD#oPzB=4qR}p_0Qo0c5 zJFxrYQ(?Zshd5N42g~4~r;m(9G)Zl72`>-9)J_n8#);)jq{ZEDr|S?;<37xZ?~bK7 zk)m^Rb+fq{+Q)INqfuI+FtBHVY9oCI(=EX2Qq$`YPz~mdSy?$&zKwI4o+ZL3^j4RX zIE3}U-MJ=Xx^t}D`PXHqgb9VkdT~0fz#It*3T#JrW5DR+!1kfV-F?T7ZniwKS)qUS z@we&bNeERk=g53Cf0=t1S)PvygST)gX;$E{J~S|ncu5*WF!NK|6*Ev|HDk)lmlU-f z3&yf{Zd85N>D5K6MKJ{3U^R_D_fXqX7<~}TCy4KN=i^47TKfasR1AK$vY~dXj}D z9IJHO+B1^QKO67L`xrI@z7eO2m&)vSLJki-qJcL$p#}yraYzq3@JT7=Kl-HZ0Ow^@ z&-PlTA+L(kWoU55N}I7zu^eqE;fV-g)x5$R^1#BgKO07hK6z4u61!Xq0ISXWHkmD8X*O>n`c)KlDd#2ecN_Kz7s; zogtG%tFbRT;e+Z3$)fRu46RZFb(>QZ%hj{?4q|tiU?w|NK;Uy^unRxUT?La`uI%C-ZIz$ zdNk+W989wTJitleK46}E+E*3vQlQo@MOFR3HcRHQCJG0_*ao!HtUwIsgCrtI?3srKf;n^O!#o4(H7 zzga;VM!viPbf8h|Kuo2KcF2nhBMDuph5}{!HDJla545O#nEyOHSu1=%akj7aif~^S zNg^LEWZ8#Du&u1wIVS!)mT1q>w$8DAA!{4H|K`?ekYNriRl+eZ06yH<3~r!`Z&Ol$ zn~_}E(@H5AvMg2Ft%y+978k=2wXE+I?CvQ%hA!lP`7;#_?{ops^2RLhZ+~V2gT;p` zei2L)bk5p4NmvboD#MtPfJysD|Dl{f$)NJP)GMT#Laj3KUW( zmiD(Si96&tJMRB7XPV0XO91*)Ooq<ae;*S_spFT6K#z4LtRFcA9iWDeDjeVI^p9*zb;`HxdVq2DnnH zX0!#P6$K(T6Byd-;1|bG{|%IHBFG>4a;udoE{zHOyKwEXtt$&ILK>k&M7CgckFmof z{5E8G+ZJN9ab;z~z#FdE5E|TolRe&WbT~b^0w!Tx#+Pw)9B7>huER#EBnt`nPa~vv zb91bMx3jUc0AM3!>C{?y$@5f>h_zUg3HRoWWGoqV4nrd)JK<8uQjwRXd-m1%clc1v zzl(dVLRaF)dWc1h;AG3R7`zby{>X#yxN9qNo=~K|oWJR9K#EcY`);hNtmN(}Q2cs| z%1L3ZkdOFSre88ro`86?*jP<9Cb0O*gL)y%AbPV}?|(dPAVhit=)$$!$P9W4Lpwp) ziCQ1AuFcA%7hXO}1%6jfH~p8Dj=&(P5RSZyaD1k^C!IS^i}jL*%fbqWw_!WDVo3h+ zj)U>@ThNsfLRU)BnbVClke~&er4g*0(5J%xV37{xWozSuT~>gIDqx|ZeaT1Z{Cwm+ zWYDR#OhGkpR-%aWH?dp-Fa0JN-u1z!UUrO-Do}<*%-?l5!A|!T-8d7^59|ThMLrqb zK&r6c;d)KK3TSu4?ZTEqUlr|z@Id_Nim3C7*CE!UTBp0*-X1OkSjWnIKv8YF2Pm0581yiBE*UC> z?SBk1tWspK1fd<$f6H1BsdmJVT!Y^&LkSrgWX_Vd$T. */ -package typeutils +// Package ap contains models and utilities for working with activitypub/activitystreams representations. +// +// It is built on top of go-fed/activity. +package ap import ( "crypto/rsa" @@ -33,7 +36,8 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/util" ) -func extractPreferredUsername(i withPreferredUsername) (string, error) { +// ExtractPreferredUsername returns a string representation of an interface's preferredUsername property. +func ExtractPreferredUsername(i WithPreferredUsername) (string, error) { u := i.GetActivityStreamsPreferredUsername() if u == nil || !u.IsXMLSchemaString() { return "", errors.New("preferredUsername was not a string") @@ -44,7 +48,8 @@ func extractPreferredUsername(i withPreferredUsername) (string, error) { return u.GetXMLSchemaString(), nil } -func extractName(i withName) (string, error) { +// ExtractName returns a string representation of an interface's name property. +func ExtractName(i WithName) (string, error) { nameProp := i.GetActivityStreamsName() if nameProp == nil { return "", errors.New("activityStreamsName not found") @@ -60,22 +65,42 @@ func extractName(i withName) (string, error) { return "", errors.New("activityStreamsName not found") } -func extractInReplyToURI(i withInReplyTo) (*url.URL, error) { +// ExtractInReplyToURI extracts the inReplyToURI property (if present) from an interface. +func ExtractInReplyToURI(i WithInReplyTo) *url.URL { inReplyToProp := i.GetActivityStreamsInReplyTo() if inReplyToProp == nil { - return nil, errors.New("in reply to prop was nil") + // the property just wasn't set + return nil } for iter := inReplyToProp.Begin(); iter != inReplyToProp.End(); iter = iter.Next() { if iter.IsIRI() { if iter.GetIRI() != nil { - return iter.GetIRI(), nil + return iter.GetIRI() } } } - return nil, errors.New("couldn't find iri for in reply to") + // couldn't find a URI + return nil } -func extractTos(i withTo) ([]*url.URL, error) { +// ExtractURLItems extracts a slice of URLs from a property that has withItems. +func ExtractURLItems(i WithItems) []*url.URL { + urls := []*url.URL{} + items := i.GetActivityStreamsItems() + if items == nil || items.Len() == 0 { + return urls + } + + for iter := items.Begin(); iter != items.End(); iter = iter.Next() { + if iter.IsIRI() { + urls = append(urls, iter.GetIRI()) + } + } + return urls +} + +// ExtractTos returns a list of URIs that the activity addresses as To. +func ExtractTos(i WithTo) ([]*url.URL, error) { to := []*url.URL{} toProp := i.GetActivityStreamsTo() if toProp == nil { @@ -91,7 +116,8 @@ func extractTos(i withTo) ([]*url.URL, error) { return to, nil } -func extractCCs(i withCC) ([]*url.URL, error) { +// ExtractCCs returns a list of URIs that the activity addresses as CC. +func ExtractCCs(i WithCC) ([]*url.URL, error) { cc := []*url.URL{} ccProp := i.GetActivityStreamsCc() if ccProp == nil { @@ -107,7 +133,8 @@ func extractCCs(i withCC) ([]*url.URL, error) { return cc, nil } -func extractAttributedTo(i withAttributedTo) (*url.URL, error) { +// ExtractAttributedTo returns the URL of the actor that the withAttributedTo is attributed to. +func ExtractAttributedTo(i WithAttributedTo) (*url.URL, error) { attributedToProp := i.GetActivityStreamsAttributedTo() if attributedToProp == nil { return nil, errors.New("attributedToProp was nil") @@ -122,7 +149,8 @@ func extractAttributedTo(i withAttributedTo) (*url.URL, error) { return nil, errors.New("couldn't find iri for attributed to") } -func extractPublished(i withPublished) (time.Time, error) { +// ExtractPublished extracts the publication time of an activity. +func ExtractPublished(i WithPublished) (time.Time, error) { publishedProp := i.GetActivityStreamsPublished() if publishedProp == nil { return time.Time{}, errors.New("published prop was nil") @@ -139,13 +167,13 @@ func extractPublished(i withPublished) (time.Time, error) { return t, nil } -// extractIconURL extracts a URL to a supported image file from something like: +// ExtractIconURL extracts a URL to a supported image file from something like: // "icon": { // "mediaType": "image/jpeg", // "type": "Image", // "url": "http://example.org/path/to/some/file.jpeg" // }, -func extractIconURL(i withIcon) (*url.URL, error) { +func ExtractIconURL(i WithIcon) (*url.URL, error) { iconProp := i.GetActivityStreamsIcon() if iconProp == nil { return nil, errors.New("icon property was nil") @@ -166,7 +194,7 @@ func extractIconURL(i withIcon) (*url.URL, error) { } // 2. has a URL so we can grab it - url, err := extractURL(imageValue) + url, err := ExtractURL(imageValue) if err == nil && url != nil { return url, nil } @@ -175,13 +203,13 @@ func extractIconURL(i withIcon) (*url.URL, error) { return nil, errors.New("could not extract valid image from icon") } -// extractImageURL extracts a URL to a supported image file from something like: +// ExtractImageURL extracts a URL to a supported image file from something like: // "image": { // "mediaType": "image/jpeg", // "type": "Image", // "url": "http://example.org/path/to/some/file.jpeg" // }, -func extractImageURL(i withImage) (*url.URL, error) { +func ExtractImageURL(i WithImage) (*url.URL, error) { imageProp := i.GetActivityStreamsImage() if imageProp == nil { return nil, errors.New("icon property was nil") @@ -202,7 +230,7 @@ func extractImageURL(i withImage) (*url.URL, error) { } // 2. has a URL so we can grab it - url, err := extractURL(imageValue) + url, err := ExtractURL(imageValue) if err == nil && url != nil { return url, nil } @@ -211,7 +239,8 @@ func extractImageURL(i withImage) (*url.URL, error) { return nil, errors.New("could not extract valid image from image property") } -func extractSummary(i withSummary) (string, error) { +// ExtractSummary extracts the summary/content warning of an interface. +func ExtractSummary(i WithSummary) (string, error) { summaryProp := i.GetActivityStreamsSummary() if summaryProp == nil { return "", errors.New("summary property was nil") @@ -226,14 +255,16 @@ func extractSummary(i withSummary) (string, error) { return "", errors.New("could not extract summary") } -func extractDiscoverable(i withDiscoverable) (bool, error) { +// ExtractDiscoverable extracts the Discoverable boolean of an interface. +func ExtractDiscoverable(i WithDiscoverable) (bool, error) { if i.GetTootDiscoverable() == nil { return false, errors.New("discoverable was nil") } return i.GetTootDiscoverable().Get(), nil } -func extractURL(i withURL) (*url.URL, error) { +// ExtractURL extracts the URL property of an interface. +func ExtractURL(i WithURL) (*url.URL, error) { urlProp := i.GetActivityStreamsUrl() if urlProp == nil { return nil, errors.New("url property was nil") @@ -248,7 +279,9 @@ func extractURL(i withURL) (*url.URL, error) { return nil, errors.New("could not extract url") } -func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKey, *url.URL, error) { +// ExtractPublicKeyForOwner extracts the public key from an interface, as long as it belongs to the specified owner. +// It will return the public key itself, the id/URL of the public key, or an error if something goes wrong. +func ExtractPublicKeyForOwner(i WithPublicKey, forOwner *url.URL) (*rsa.PublicKey, *url.URL, error) { publicKeyProp := i.GetW3IDSecurityV1PublicKey() if publicKeyProp == nil { return nil, nil, errors.New("public key property was nil") @@ -298,7 +331,8 @@ func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKe return nil, nil, errors.New("couldn't find public key") } -func extractContent(i withContent) (string, error) { +// ExtractContent returns a string representation of the interface's Content property. +func ExtractContent(i WithContent) (string, error) { contentProperty := i.GetActivityStreamsContent() if contentProperty == nil { return "", errors.New("content property was nil") @@ -311,7 +345,8 @@ func extractContent(i withContent) (string, error) { return "", errors.New("no content found") } -func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) { +// ExtractAttachments returns a slice of attachments on the interface. +func ExtractAttachments(i WithAttachment) ([]*gtsmodel.MediaAttachment, error) { attachments := []*gtsmodel.MediaAttachment{} attachmentProp := i.GetActivityStreamsAttachment() if attachmentProp == nil { @@ -326,7 +361,7 @@ func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) { if !ok { continue } - attachment, err := extractAttachment(attachmentable) + attachment, err := ExtractAttachment(attachmentable) if err != nil { continue } @@ -335,12 +370,13 @@ func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) { return attachments, nil } -func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { +// ExtractAttachment returns a gts model of an attachment from an attachmentable interface. +func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { attachment := >smodel.MediaAttachment{ File: gtsmodel.File{}, } - attachmentURL, err := extractURL(i) + attachmentURL, err := ExtractURL(i) if err != nil { return nil, err } @@ -356,7 +392,7 @@ func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { attachment.File.ContentType = mediaType.Get() attachment.Type = gtsmodel.FileTypeImage - name, err := extractName(i) + name, err := ExtractName(i) if err == nil { attachment.Description = name } @@ -376,7 +412,8 @@ func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) { // return i.GetTootBlurhashProperty().Get(), nil // } -func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) { +// ExtractHashtags returns a slice of tags on the interface. +func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) { tags := []*gtsmodel.Tag{} tagsProp := i.GetActivityStreamsTag() if tagsProp == nil { @@ -397,7 +434,7 @@ func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) { continue } - tag, err := extractHashtag(hashtaggable) + tag, err := ExtractHashtag(hashtaggable) if err != nil { continue } @@ -407,7 +444,8 @@ func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) { return tags, nil } -func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { +// ExtractHashtag returns a gtsmodel tag from a hashtaggable. +func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { tag := >smodel.Tag{} hrefProp := i.GetActivityStreamsHref() @@ -416,7 +454,7 @@ func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { } tag.URL = hrefProp.GetIRI().String() - name, err := extractName(i) + name, err := ExtractName(i) if err != nil { return nil, err } @@ -425,7 +463,8 @@ func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) { return tag, nil } -func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) { +// ExtractEmojis returns a slice of emojis on the interface. +func ExtractEmojis(i WithTag) ([]*gtsmodel.Emoji, error) { emojis := []*gtsmodel.Emoji{} tagsProp := i.GetActivityStreamsTag() if tagsProp == nil { @@ -446,7 +485,7 @@ func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) { continue } - emoji, err := extractEmoji(emojiable) + emoji, err := ExtractEmoji(emojiable) if err != nil { continue } @@ -456,7 +495,8 @@ func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) { return emojis, nil } -func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { +// ExtractEmoji ... +func ExtractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { emoji := >smodel.Emoji{} idProp := i.GetJSONLDId() @@ -467,7 +507,7 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { emoji.URI = uri.String() emoji.Domain = uri.Host - name, err := extractName(i) + name, err := ExtractName(i) if err != nil { return nil, err } @@ -476,7 +516,7 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { if i.GetActivityStreamsIcon() == nil { return nil, errors.New("no icon for emoji") } - imageURL, err := extractIconURL(i) + imageURL, err := ExtractIconURL(i) if err != nil { return nil, errors.New("no url for emoji image") } @@ -485,7 +525,8 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) { return emoji, nil } -func extractMentions(i withTag) ([]*gtsmodel.Mention, error) { +// ExtractMentions extracts a slice of gtsmodel Mentions from a WithTag interface. +func ExtractMentions(i WithTag) ([]*gtsmodel.Mention, error) { mentions := []*gtsmodel.Mention{} tagsProp := i.GetActivityStreamsTag() if tagsProp == nil { @@ -506,7 +547,7 @@ func extractMentions(i withTag) ([]*gtsmodel.Mention, error) { continue } - mention, err := extractMention(mentionable) + mention, err := ExtractMention(mentionable) if err != nil { continue } @@ -516,10 +557,11 @@ func extractMentions(i withTag) ([]*gtsmodel.Mention, error) { return mentions, nil } -func extractMention(i Mentionable) (*gtsmodel.Mention, error) { +// ExtractMention extracts a gts model mention from a Mentionable. +func ExtractMention(i Mentionable) (*gtsmodel.Mention, error) { mention := >smodel.Mention{} - mentionString, err := extractName(i) + mentionString, err := ExtractName(i) if err != nil { return nil, err } @@ -543,7 +585,8 @@ func extractMention(i Mentionable) (*gtsmodel.Mention, error) { return mention, nil } -func extractActor(i withActor) (*url.URL, error) { +// ExtractActor extracts the actor ID/IRI from an interface WithActor. +func ExtractActor(i WithActor) (*url.URL, error) { actorProp := i.GetActivityStreamsActor() if actorProp == nil { return nil, errors.New("actor property was nil") @@ -556,7 +599,8 @@ func extractActor(i withActor) (*url.URL, error) { return nil, errors.New("no iri found for actor prop") } -func extractObject(i withObject) (*url.URL, error) { +// ExtractObject extracts a URL object from a WithObject interface. +func ExtractObject(i WithObject) (*url.URL, error) { objectProp := i.GetActivityStreamsObject() if objectProp == nil { return nil, errors.New("object property was nil") diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go new file mode 100644 index 000000000..43dd149d5 --- /dev/null +++ b/internal/ap/interfaces.go @@ -0,0 +1,321 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package ap + +import "github.com/go-fed/activity/streams/vocab" + +// Accountable represents the minimum activitypub interface for representing an 'account'. +// This interface is fulfilled by: Person, Application, Organization, Service, and Group +type Accountable interface { + WithJSONLDId + WithTypeName + + WithPreferredUsername + WithIcon + WithName + WithImage + WithSummary + WithDiscoverable + WithURL + WithPublicKey + WithInbox + WithOutbox + WithFollowing + WithFollowers + WithFeatured +} + +// Statusable represents the minimum activitypub interface for representing a 'status'. +// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile +type Statusable interface { + WithJSONLDId + WithTypeName + + WithSummary + WithInReplyTo + WithPublished + WithURL + WithAttributedTo + WithTo + WithCC + WithSensitive + WithConversation + WithContent + WithAttachment + WithTag + WithReplies +} + +// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'. +// This interface is fulfilled by: Audio, Document, Image, Video +type Attachmentable interface { + WithTypeName + WithMediaType + WithURL + WithName +} + +// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag. +type Hashtaggable interface { + WithTypeName + WithHref + WithName +} + +// Emojiable represents the minimum interface for an 'emoji' tag. +type Emojiable interface { + WithJSONLDId + WithTypeName + WithName + WithUpdated + WithIcon +} + +// Mentionable represents the minimum interface for a 'mention' tag. +type Mentionable interface { + WithName + WithHref +} + +// Followable represents the minimum interface for an activitystreams 'follow' activity. +type Followable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject +} + +// Likeable represents the minimum interface for an activitystreams 'like' activity. +type Likeable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject +} + +// Blockable represents the minimum interface for an activitystreams 'block' activity. +type Blockable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject +} + +// Announceable represents the minimum interface for an activitystreams 'announce' activity. +type Announceable interface { + WithJSONLDId + WithTypeName + + WithActor + WithObject + WithPublished + WithTo + WithCC +} + +// CollectionPageable represents the minimum interface for an activitystreams 'CollectionPage' object. +type CollectionPageable interface { + WithJSONLDId + WithTypeName + + WithNext + WithPartOf + WithItems +} + +// WithJSONLDId represents an activity with JSONLDIdProperty +type WithJSONLDId interface { + GetJSONLDId() vocab.JSONLDIdProperty +} + +// WithTypeName represents an activity with a type name +type WithTypeName interface { + GetTypeName() string +} + +// WithPreferredUsername represents an activity with ActivityStreamsPreferredUsernameProperty +type WithPreferredUsername interface { + GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty +} + +// WithIcon represents an activity with ActivityStreamsIconProperty +type WithIcon interface { + GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty +} + +// WithName represents an activity with ActivityStreamsNameProperty +type WithName interface { + GetActivityStreamsName() vocab.ActivityStreamsNameProperty +} + +// WithImage represents an activity with ActivityStreamsImageProperty +type WithImage interface { + GetActivityStreamsImage() vocab.ActivityStreamsImageProperty +} + +// WithSummary represents an activity with ActivityStreamsSummaryProperty +type WithSummary interface { + GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty +} + +// WithDiscoverable represents an activity with TootDiscoverableProperty +type WithDiscoverable interface { + GetTootDiscoverable() vocab.TootDiscoverableProperty +} + +// WithURL represents an activity with ActivityStreamsUrlProperty +type WithURL interface { + GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty +} + +// WithPublicKey represents an activity with W3IDSecurityV1PublicKeyProperty +type WithPublicKey interface { + GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty +} + +// WithInbox represents an activity with ActivityStreamsInboxProperty +type WithInbox interface { + GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty +} + +// WithOutbox represents an activity with ActivityStreamsOutboxProperty +type WithOutbox interface { + GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty +} + +// WithFollowing represents an activity with ActivityStreamsFollowingProperty +type WithFollowing interface { + GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty +} + +// WithFollowers represents an activity with ActivityStreamsFollowersProperty +type WithFollowers interface { + GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty +} + +// WithFeatured represents an activity with TootFeaturedProperty +type WithFeatured interface { + GetTootFeatured() vocab.TootFeaturedProperty +} + +// WithAttributedTo represents an activity with ActivityStreamsAttributedToProperty +type WithAttributedTo interface { + GetActivityStreamsAttributedTo() vocab.ActivityStreamsAttributedToProperty +} + +// WithAttachment represents an activity with ActivityStreamsAttachmentProperty +type WithAttachment interface { + GetActivityStreamsAttachment() vocab.ActivityStreamsAttachmentProperty +} + +// WithTo represents an activity with ActivityStreamsToProperty +type WithTo interface { + GetActivityStreamsTo() vocab.ActivityStreamsToProperty +} + +// WithInReplyTo represents an activity with ActivityStreamsInReplyToProperty +type WithInReplyTo interface { + GetActivityStreamsInReplyTo() vocab.ActivityStreamsInReplyToProperty +} + +// WithCC represents an activity with ActivityStreamsCcProperty +type WithCC interface { + GetActivityStreamsCc() vocab.ActivityStreamsCcProperty +} + +// WithSensitive ... +type WithSensitive interface { + // TODO +} + +// WithConversation ... +type WithConversation interface { + // TODO +} + +// WithContent represents an activity with ActivityStreamsContentProperty +type WithContent interface { + GetActivityStreamsContent() vocab.ActivityStreamsContentProperty +} + +// WithPublished represents an activity with ActivityStreamsPublishedProperty +type WithPublished interface { + GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty +} + +// WithTag represents an activity with ActivityStreamsTagProperty +type WithTag interface { + GetActivityStreamsTag() vocab.ActivityStreamsTagProperty +} + +// WithReplies represents an activity with ActivityStreamsRepliesProperty +type WithReplies interface { + GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty +} + +// WithMediaType represents an activity with ActivityStreamsMediaTypeProperty +type WithMediaType interface { + GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty +} + +// type withBlurhash interface { +// GetTootBlurhashProperty() vocab.TootBlurhashProperty +// } + +// type withFocalPoint interface { +// // TODO +// } + +// WithHref represents an activity with ActivityStreamsHrefProperty +type WithHref interface { + GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty +} + +// WithUpdated represents an activity with ActivityStreamsUpdatedProperty +type WithUpdated interface { + GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty +} + +// WithActor represents an activity with ActivityStreamsActorProperty +type WithActor interface { + GetActivityStreamsActor() vocab.ActivityStreamsActorProperty +} + +// WithObject represents an activity with ActivityStreamsObjectProperty +type WithObject interface { + GetActivityStreamsObject() vocab.ActivityStreamsObjectProperty +} + +// WithNext represents an activity with ActivityStreamsNextProperty +type WithNext interface { + GetActivityStreamsNext() vocab.ActivityStreamsNextProperty +} + +// WithPartOf represents an activity with ActivityStreamsPartOfProperty +type WithPartOf interface { + GetActivityStreamsPartOf() vocab.ActivityStreamsPartOfProperty +} + +// WithItems represents an activity with ActivityStreamsItemsProperty +type WithItems interface { + GetActivityStreamsItems() vocab.ActivityStreamsItemsProperty +} diff --git a/internal/api/client/account/accountupdate_test.go b/internal/api/client/account/accountupdate_test.go index 341b865ff..349429625 100644 --- a/internal/api/client/account/accountupdate_test.go +++ b/internal/api/client/account/accountupdate_test.go @@ -53,10 +53,10 @@ func (suite *AccountUpdateTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.accountModule = account.New(suite.config, suite.processor, suite.log).(*account.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } @@ -80,6 +80,8 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandler() ctx, _ := gin.CreateTestContext(recorder) ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"]) ctx.Set(oauth.SessionAuthorizedToken, oauth.TokenToOauthToken(suite.testTokens["local_account_1"])) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"]) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"]) ctx.Request = httptest.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:8080/%s", account.UpdateCredentialsPath), bytes.NewReader(requestBody.Bytes())) // the endpoint we're hitting ctx.Request.Header.Set("Content-Type", w.FormDataContentType()) suite.accountModule.AccountUpdateCredentialsPATCHHandler(ctx) diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go index cb503facb..4eec3bae5 100644 --- a/internal/api/client/fileserver/servefile_test.go +++ b/internal/api/client/fileserver/servefile_test.go @@ -78,7 +78,7 @@ func (suite *ServeFileTestSuite) SetupSuite() { suite.db = testrig.NewTestDB() suite.log = testrig.NewTestLog() suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) @@ -95,7 +95,7 @@ func (suite *ServeFileTestSuite) TearDownSuite() { } func (suite *ServeFileTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 89a77a729..a61a36324 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -84,7 +84,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) // setup module being tested @@ -98,7 +98,7 @@ func (suite *MediaCreateTestSuite) TearDownSuite() { } func (suite *MediaCreateTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() diff --git a/internal/api/client/status/statusboost_test.go b/internal/api/client/status/statusboost_test.go index 9400aeddc..fbe267fac 100644 --- a/internal/api/client/status/statusboost_test.go +++ b/internal/api/client/status/statusboost_test.go @@ -52,10 +52,10 @@ func (suite *StatusBoostTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statuscreate_test.go b/internal/api/client/status/statuscreate_test.go index dd4a4386b..603432724 100644 --- a/internal/api/client/status/statuscreate_test.go +++ b/internal/api/client/status/statuscreate_test.go @@ -58,10 +58,10 @@ func (suite *StatusCreateTestSuite) SetupTest() { suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusfave_test.go b/internal/api/client/status/statusfave_test.go index b1cafc2fb..0f44b5e90 100644 --- a/internal/api/client/status/statusfave_test.go +++ b/internal/api/client/status/statusfave_test.go @@ -55,10 +55,10 @@ func (suite *StatusFaveTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusfavedby_test.go b/internal/api/client/status/statusfavedby_test.go index b6e1591e0..22a549b30 100644 --- a/internal/api/client/status/statusfavedby_test.go +++ b/internal/api/client/status/statusfavedby_test.go @@ -55,10 +55,10 @@ func (suite *StatusFavedByTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusget_test.go b/internal/api/client/status/statusget_test.go index 1bbf48a91..1c700aaa5 100644 --- a/internal/api/client/status/statusget_test.go +++ b/internal/api/client/status/statusget_test.go @@ -45,10 +45,10 @@ func (suite *StatusGetTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/client/status/statusunfave_test.go b/internal/api/client/status/statusunfave_test.go index 36144c5ce..a5f267f4c 100644 --- a/internal/api/client/status/statusunfave_test.go +++ b/internal/api/client/status/statusunfave_test.go @@ -55,10 +55,10 @@ func (suite *StatusUnfaveTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } diff --git a/internal/api/s2s/user/repliesget.go b/internal/api/s2s/user/repliesget.go new file mode 100644 index 000000000..951cc428c --- /dev/null +++ b/internal/api/s2s/user/repliesget.go @@ -0,0 +1,186 @@ +package user + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// StatusRepliesGETHandler swagger:operation GET /users/{username}/statuses/{status}/replies s2sRepliesGet +// +// Get the replies collection for a status. +// +// Note that the response will be a Collection with a page as `first`, as shown below, if `page` is `false`. +// +// If `page` is `true`, then the response will be a single `CollectionPage` without the wrapping `Collection`. +// +// HTTP signature is required on the request. +// +// --- +// tags: +// - s2s/federation +// +// produces: +// - application/activity+json +// +// parameters: +// - name: username +// type: string +// description: Username of the account. +// in: path +// required: true +// - name: status +// type: string +// description: ID of the status. +// in: path +// required: true +// - name: page +// type: boolean +// description: Return response as a CollectionPage. +// in: query +// default: false +// - name: only_other_accounts +// type: boolean +// description: Return replies only from accounts other than the status owner. +// in: query +// default: false +// - name: min_id +// type: string +// description: Minimum ID of the next status, used for paging. +// in: query +// +// responses: +// '200': +// in: body +// schema: +// "$ref": "#/definitions/swaggerStatusRepliesCollection" +// '400': +// description: bad request +// '401': +// description: unauthorized +// '403': +// description: forbidden +// '404': +// description: not found +func (m *Module) StatusRepliesGETHandler(c *gin.Context) { + l := m.log.WithFields(logrus.Fields{ + "func": "StatusRepliesGETHandler", + "url": c.Request.RequestURI, + }) + + requestedUsername := c.Param(UsernameKey) + if requestedUsername == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "no username specified in request"}) + return + } + + requestedStatusID := c.Param(StatusIDKey) + if requestedStatusID == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "no status id specified in request"}) + return + } + + page := false + pageString := c.Query(PageKey) + if pageString != "" { + i, err := strconv.ParseBool(pageString) + if err != nil { + l.Debugf("error parsing page string: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse page query param"}) + return + } + page = i + } + + onlyOtherAccounts := false + onlyOtherAccountsString := c.Query(OnlyOtherAccountsKey) + if onlyOtherAccountsString != "" { + i, err := strconv.ParseBool(onlyOtherAccountsString) + if err != nil { + l.Debugf("error parsing only_other_accounts string: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse only_other_accounts query param"}) + return + } + onlyOtherAccounts = i + } + + minID := "" + minIDString := c.Query(MinIDKey) + if minIDString != "" { + minID = minIDString + } + + // make sure this actually an AP request + format := c.NegotiateFormat(ActivityPubAcceptHeaders...) + if format == "" { + c.JSON(http.StatusNotAcceptable, gin.H{"error": "could not negotiate format with given Accept header(s)"}) + return + } + l.Tracef("negotiated format: %s", format) + + // transfer the signature verifier from the gin context to the request context + ctx := c.Request.Context() + verifier, signed := c.Get(string(util.APRequestingPublicKeyVerifier)) + if signed { + ctx = context.WithValue(ctx, util.APRequestingPublicKeyVerifier, verifier) + } + + replies, err := m.processor.GetFediStatusReplies(ctx, requestedUsername, requestedStatusID, page, onlyOtherAccounts, minID, c.Request.URL) + if err != nil { + l.Info(err.Error()) + c.JSON(err.Code(), gin.H{"error": err.Safe()}) + return + } + + b, mErr := json.Marshal(replies) + if mErr != nil { + err := fmt.Errorf("could not marshal json: %s", mErr) + l.Error(err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + c.Data(http.StatusOK, format, b) +} + +// SwaggerStatusRepliesCollection represents a response to GET /users/{username}/statuses/{status}/replies. +// swagger:model swaggerStatusRepliesCollection +type SwaggerStatusRepliesCollection struct { + // ActivityStreams context. + // example: https://www.w3.org/ns/activitystreams + Context string `json:"@context"` + // ActivityStreams ID. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies + ID string `json:"id"` + // ActivityStreams type. + // example: Collection + Type string `json:"type"` + // ActivityStreams first property. + First SwaggerStatusRepliesCollectionPage `json:"first"` +} + +// SwaggerStatusRepliesCollectionPage represents one page of a collection. +// swagger:model swaggerStatusRepliesCollectionPage +type SwaggerStatusRepliesCollectionPage struct { + // ActivityStreams ID. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true + ID string `json:"id"` + // ActivityStreams type. + // example: CollectionPage + Type string `json:"type"` + // Link to the next page. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true + Next string `json:"next"` + // Collection this page belongs to. + // example: https://example.org/users/some_user/statuses/106717595988259568/replies + PartOf string `json:"partOf"` + // Items on this page. + // example: ["https://example.org/users/some_other_user/statuses/086417595981111564", "https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R"] + Items []string `json:"items"` +} diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go new file mode 100644 index 000000000..75edbc882 --- /dev/null +++ b/internal/api/s2s/user/repliesget_test.go @@ -0,0 +1,241 @@ +package user_test + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/api/security" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type RepliesGetTestSuite struct { + UserStandardTestSuite +} + +func (suite *RepliesGetTestSuite) SetupSuite() { + suite.testTokens = testrig.NewTestTokens() + suite.testClients = testrig.NewTestClients() + suite.testApplications = testrig.NewTestApplications() + suite.testUsers = testrig.NewTestUsers() + suite.testAccounts = testrig.NewTestAccounts() + suite.testAttachments = testrig.NewTestAttachments() + suite.testStatuses = testrig.NewTestStatuses() +} + +func (suite *RepliesGetTestSuite) SetupTest() { + suite.config = testrig.NewTestConfig() + suite.db = testrig.NewTestDB() + suite.tc = testrig.NewTestTypeConverter(suite.db) + suite.storage = testrig.NewTestStorage() + suite.log = testrig.NewTestLog() + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) + suite.userModule = user.New(suite.config, suite.processor, suite.log).(*user.Module) + suite.securityModule = security.New(suite.config, suite.db, suite.log).(*security.Module) + testrig.StandardDBSetup(suite.db, suite.testAccounts) + testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") +} + +func (suite *RepliesGetTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) + testrig.StandardStorageTeardown(suite.storage) +} + +func (suite *RepliesGetTestSuite) TestGetReplies() { + // the dereference we're gonna use + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies"] + targetAccount := suite.testAccounts["local_account_1"] + targetStatus := suite.testStatuses["local_account_1_status_1"] + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) + userModule := user.New(suite.config, processor, suite.log).(*user.Module) + + // setup request + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies", nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) + + // normally the router would populate these params from the path values, + // but because we're calling the function directly, we need to set them manually. + ctx.Params = gin.Params{ + gin.Param{ + Key: user.UsernameKey, + Value: targetAccount.Username, + }, + gin.Param{ + Key: user.StatusIDKey, + Value: targetStatus.ID, + }, + } + + // trigger the function being tested + userModule.StatusRepliesGETHandler(ctx) + + // check response + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","first":{"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"Collection"}`, string(b)) + + // should be a Collection + m := make(map[string]interface{}) + err = json.Unmarshal(b, &m) + assert.NoError(suite.T(), err) + + t, err := streams.ToType(context.Background(), m) + assert.NoError(suite.T(), err) + + _, ok := t.(vocab.ActivityStreamsCollection) + assert.True(suite.T(), ok) +} + +func (suite *RepliesGetTestSuite) TestGetRepliesNext() { + // the dereference we're gonna use + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies_next"] + targetAccount := suite.testAccounts["local_account_1"] + targetStatus := suite.testStatuses["local_account_1_status_1"] + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) + userModule := user.New(suite.config, processor, suite.log).(*user.Module) + + // setup request + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true", nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) + + // normally the router would populate these params from the path values, + // but because we're calling the function directly, we need to set them manually. + ctx.Params = gin.Params{ + gin.Param{ + Key: user.UsernameKey, + Value: targetAccount.Username, + }, + gin.Param{ + Key: user.StatusIDKey, + Value: targetStatus.ID, + }, + } + + // trigger the function being tested + userModule.StatusRepliesGETHandler(ctx) + + // check response + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + + // should be a Collection + m := make(map[string]interface{}) + err = json.Unmarshal(b, &m) + assert.NoError(suite.T(), err) + + t, err := streams.ToType(context.Background(), m) + assert.NoError(suite.T(), err) + + page, ok := t.(vocab.ActivityStreamsCollectionPage) + assert.True(suite.T(), ok) + + assert.Equal(suite.T(), page.GetActivityStreamsItems().Len(), 1) +} + +func (suite *RepliesGetTestSuite) TestGetRepliesLast() { + // the dereference we're gonna use + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies_last"] + targetAccount := suite.testAccounts["local_account_1"] + targetStatus := suite.testStatuses["local_account_1_status_1"] + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) + userModule := user.New(suite.config, processor, suite.log).(*user.Module) + + // setup request + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) + + // normally the router would populate these params from the path values, + // but because we're calling the function directly, we need to set them manually. + ctx.Params = gin.Params{ + gin.Param{ + Key: user.UsernameKey, + Value: targetAccount.Username, + }, + gin.Param{ + Key: user.StatusIDKey, + Value: targetStatus.ID, + }, + } + + // trigger the function being tested + userModule.StatusRepliesGETHandler(ctx) + + // check response + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + + fmt.Println(string(b)) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + + // should be a Collection + m := make(map[string]interface{}) + err = json.Unmarshal(b, &m) + assert.NoError(suite.T(), err) + + t, err := streams.ToType(context.Background(), m) + assert.NoError(suite.T(), err) + + page, ok := t.(vocab.ActivityStreamsCollectionPage) + assert.True(suite.T(), ok) + + assert.Equal(suite.T(), page.GetActivityStreamsItems().Len(), 0) +} + +func TestRepliesGetTestSuite(t *testing.T) { + suite.Run(t, new(RepliesGetTestSuite)) +} diff --git a/internal/api/s2s/user/user.go b/internal/api/s2s/user/user.go index 0cb8e1e90..b5ff9a699 100644 --- a/internal/api/s2s/user/user.go +++ b/internal/api/s2s/user/user.go @@ -34,6 +34,13 @@ const ( UsernameKey = "username" // StatusIDKey is for status IDs StatusIDKey = "status" + // OnlyOtherAccountsKey is for filtering status responses. + OnlyOtherAccountsKey = "only_other_accounts" + // MinIDKey is for filtering status responses. + MinIDKey = "min_id" + // PageKey is for filtering status responses. + PageKey = "page" + // UsersBasePath is the base path for serving information about Users eg https://example.org/users UsersBasePath = "/" + util.UsersPath // UsersBasePathWithUsername is just the users base path with the Username key in it. @@ -50,6 +57,8 @@ const ( UsersFollowingPath = UsersBasePathWithUsername + "/" + util.FollowingPath // UsersStatusPath is for serving GET requests to a particular status by a user, with the given username key and status ID UsersStatusPath = UsersBasePathWithUsername + "/" + util.StatusesPath + "/:" + StatusIDKey + // UsersStatusRepliesPath is for serving the replies collection of a status. + UsersStatusRepliesPath = UsersStatusPath + "/replies" ) // ActivityPubAcceptHeaders represents the Accept headers mentioned here: @@ -83,5 +92,6 @@ func (m *Module) Route(s router.Router) error { s.AttachHandler(http.MethodGet, UsersFollowingPath, m.FollowingGETHandler) s.AttachHandler(http.MethodGet, UsersStatusPath, m.StatusGETHandler) s.AttachHandler(http.MethodGet, UsersPublicKeyPath, m.PublicKeyGETHandler) + s.AttachHandler(http.MethodGet, UsersStatusRepliesPath, m.StatusRepliesGETHandler) return nil } diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go index 91d1ea32d..71d4395eb 100644 --- a/internal/api/s2s/user/user_test.go +++ b/internal/api/s2s/user/user_test.go @@ -4,6 +4,7 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/api/security" "github.com/superseriousbusiness/gotosocial/internal/blob" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -18,13 +19,14 @@ import ( type UserStandardTestSuite struct { // standard suite interfaces suite.Suite - config *config.Config - db db.DB - log *logrus.Logger - tc typeutils.TypeConverter - federator federation.Federator - processor processing.Processor - storage blob.Storage + config *config.Config + db db.DB + log *logrus.Logger + tc typeutils.TypeConverter + federator federation.Federator + processor processing.Processor + storage blob.Storage + securityModule *security.Module // standard suite models testTokens map[string]*oauth.Token diff --git a/internal/api/s2s/user/userget_test.go b/internal/api/s2s/user/userget_test.go index d20148802..ab0015c57 100644 --- a/internal/api/s2s/user/userget_test.go +++ b/internal/api/s2s/user/userget_test.go @@ -1,16 +1,11 @@ package user_test import ( - "bytes" "context" - "crypto/x509" "encoding/json" - "encoding/pem" - "fmt" "io/ioutil" "net/http" "net/http/httptest" - "strings" "testing" "github.com/gin-gonic/gin" @@ -19,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/api/security" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -42,10 +38,11 @@ func (suite *UserGetTestSuite) SetupTest() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.log = testrig.NewTestLog() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator) suite.userModule = user.New(suite.config, suite.processor, suite.log).(*user.Module) - testrig.StandardDBSetup(suite.db) + suite.securityModule = security.New(suite.config, suite.db, suite.log).(*security.Module) + testrig.StandardDBSetup(suite.db, suite.testAccounts) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") } @@ -56,48 +53,11 @@ func (suite *UserGetTestSuite) TearDownTest() { func (suite *UserGetTestSuite) TestGetUser() { // the dereference we're gonna use - signedRequest := testrig.NewTestDereferenceRequests(suite.testAccounts)["foss_satan_dereference_zork"] - - requestingAccount := suite.testAccounts["remote_account_1"] + derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) + signedRequest := derefRequests["foss_satan_dereference_zork"] targetAccount := suite.testAccounts["local_account_1"] - encodedPublicKey, err := x509.MarshalPKIXPublicKey(requestingAccount.PublicKey) - assert.NoError(suite.T(), err) - publicKeyBytes := pem.EncodeToMemory(&pem.Block{ - Type: "PUBLIC KEY", - Bytes: encodedPublicKey, - }) - publicKeyString := strings.ReplaceAll(string(publicKeyBytes), "\n", "\\n") - - // for this test we need the client to return the public key of the requester on the 'remote' instance - responseBodyString := fmt.Sprintf(` - { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1" - ], - - "id": "%s", - "type": "Person", - "preferredUsername": "%s", - "inbox": "%s", - - "publicKey": { - "id": "%s", - "owner": "%s", - "publicKeyPem": "%s" - } - }`, requestingAccount.URI, requestingAccount.Username, requestingAccount.InboxURI, requestingAccount.PublicKeyURI, requestingAccount.URI, publicKeyString) - - // create a transport controller whose client will just return the response body string we specified above - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - r := ioutil.NopCloser(bytes.NewReader([]byte(responseBodyString))) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - })) - // get this transport controller embedded right in the user module we're testing + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) federator := testrig.NewTestFederator(suite.db, tc, suite.storage) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator) userModule := user.New(suite.config, processor, suite.log).(*user.Module) @@ -105,7 +65,12 @@ func (suite *UserGetTestSuite) TestGetUser() { // setup request recorder := httptest.NewRecorder() ctx, _ := gin.CreateTestContext(recorder) - ctx.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:8080%s", strings.Replace(user.UsersBasePathWithUsername, ":username", targetAccount.Username, 1)), nil) // the endpoint we're hitting + ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.URI, nil) // the endpoint we're hitting + ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) + ctx.Request.Header.Set("Date", signedRequest.DateHeader) + + // we need to pass the context through signature check first to set appropriate values on it + suite.securityModule.SignatureCheck(ctx) // normally the router would populate these params from the path values, // but because we're calling the function directly, we need to set them manually. @@ -116,11 +81,6 @@ func (suite *UserGetTestSuite) TestGetUser() { }, } - // we need these headers for the request to be validated - ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) - ctx.Request.Header.Set("Date", signedRequest.DateHeader) - ctx.Request.Header.Set("Digest", signedRequest.DigestHeader) - // trigger the function being tested userModule.UsersGETHandler(ctx) diff --git a/internal/cliactions/server/server.go b/internal/cliactions/server/server.go index 3c4f97dea..2314e2608 100644 --- a/internal/cliactions/server/server.go +++ b/internal/cliactions/server/server.go @@ -115,7 +115,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log // build backend handlers mediaHandler := media.New(c, dbService, storageBackend, log) oauthServer := oauth.New(dbService, log) - transportController := transport.NewController(c, &federation.Clock{}, http.DefaultClient, log) + transportController := transport.NewController(c, dbService, &federation.Clock{}, http.DefaultClient, log) federator := federation.NewFederator(dbService, federatingDB, transportController, c, log, typeConverter, mediaHandler) processor := processing.NewProcessor(c, typeConverter, federator, oauthServer, mediaHandler, storageBackend, timelineManager, dbService, log) if err := processor.Start(); err != nil { diff --git a/internal/cliactions/testrig/testrig.go b/internal/cliactions/testrig/testrig.go index e2b97fe61..a7032825c 100644 --- a/internal/cliactions/testrig/testrig.go +++ b/internal/cliactions/testrig/testrig.go @@ -46,7 +46,7 @@ import ( var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log *logrus.Logger) error { c := testrig.NewTestConfig() dbService := testrig.NewTestDB() - testrig.StandardDBSetup(dbService) + testrig.StandardDBSetup(dbService, nil) router := testrig.NewTestRouter(dbService) storageBackend := testrig.NewTestStorage() testrig.StandardStorageSetup(storageBackend, "./testrig/media") @@ -59,7 +59,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log StatusCode: 200, Body: r, }, nil - })) + }), dbService) federator := testrig.NewTestFederator(dbService, transportController, storageBackend) processor := testrig.NewTestProcessor(dbService, storageBackend, federator) diff --git a/internal/db/db.go b/internal/db/db.go index c764cc716..d0b23fbc6 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -218,10 +218,14 @@ type DB interface { GetFaveCountForStatus(status *gtsmodel.Status) (int, error) // StatusParents get the parent statuses of a given status. - StatusParents(status *gtsmodel.Status) ([]*gtsmodel.Status, error) + // + // If onlyDirect is true, only the immediate parent will be returned. + StatusParents(status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, error) // StatusChildren gets the child statuses of a given status. - StatusChildren(status *gtsmodel.Status) ([]*gtsmodel.Status, error) + // + // If onlyDirect is true, only the immediate children will be returned. + StatusChildren(status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error) // StatusFavedBy checks if a given status has been faved by a given account ID StatusFavedBy(status *gtsmodel.Status, accountID string) (bool, error) diff --git a/internal/db/pg/statuscontext.go b/internal/db/pg/statuscontext.go index 732485ab5..2ff1a20bb 100644 --- a/internal/db/pg/statuscontext.go +++ b/internal/db/pg/statuscontext.go @@ -25,14 +25,14 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (ps *postgresService) StatusParents(status *gtsmodel.Status) ([]*gtsmodel.Status, error) { +func (ps *postgresService) StatusParents(status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, error) { parents := []*gtsmodel.Status{} - ps.statusParent(status, &parents) + ps.statusParent(status, &parents, onlyDirect) return parents, nil } -func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses *[]*gtsmodel.Status) { +func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses *[]*gtsmodel.Status, onlyDirect bool) { if status.InReplyToID == "" { return } @@ -42,13 +42,16 @@ func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses * *foundStatuses = append(*foundStatuses, parentStatus) } - ps.statusParent(parentStatus, foundStatuses) + if onlyDirect { + return + } + ps.statusParent(parentStatus, foundStatuses, false) } -func (ps *postgresService) StatusChildren(status *gtsmodel.Status) ([]*gtsmodel.Status, error) { +func (ps *postgresService) StatusChildren(status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error) { foundStatuses := &list.List{} foundStatuses.PushFront(status) - ps.statusChildren(status, foundStatuses) + ps.statusChildren(status, foundStatuses, onlyDirect, minID) children := []*gtsmodel.Status{} for e := foundStatuses.Front(); e != nil; e = e.Next() { @@ -66,11 +69,15 @@ func (ps *postgresService) StatusChildren(status *gtsmodel.Status) ([]*gtsmodel. return children, nil } -func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses *list.List) { +func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses *list.List, onlyDirect bool, minID string) { immediateChildren := []*gtsmodel.Status{} - err := ps.conn.Model(&immediateChildren).Where("in_reply_to_id = ?", status.ID).Select() - if err != nil { + q := ps.conn.Model(&immediateChildren).Where("in_reply_to_id = ?", status.ID) + if minID != "" { + q = q.Where("status.id > ?", minID) + } + + if err := q.Select(); err != nil { return } @@ -88,6 +95,10 @@ func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses } } - ps.statusChildren(child, foundStatuses) + // only do one loop if we only want direct children + if onlyDirect { + return + } + ps.statusChildren(child, foundStatuses, false, minID) } } diff --git a/internal/federation/authenticate.go b/internal/federation/authenticate.go index 0cb8db6dc..699691ca6 100644 --- a/internal/federation/authenticate.go +++ b/internal/federation/authenticate.go @@ -147,6 +147,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU if strings.EqualFold(requestingHost, f.config.Host) { // LOCAL ACCOUNT REQUEST // the request is coming from INSIDE THE HOUSE so skip the remote dereferencing + l.Tracef("proceeding without dereference for local public key %s", requestingPublicKeyID) if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingLocalAccount); err != nil { return nil, false, fmt.Errorf("couldn't get local account with public key uri %s from the database: %s", requestingPublicKeyID.String(), err) } @@ -158,6 +159,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU } else if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingRemoteAccount); err == nil { // REMOTE ACCOUNT REQUEST WITH KEY CACHED LOCALLY // this is a remote account and we already have the public key for it so use that + l.Tracef("proceeding without dereference for cached public key %s", requestingPublicKeyID) publicKey = requestingRemoteAccount.PublicKey pkOwnerURI, err = url.Parse(requestingRemoteAccount.URI) if err != nil { @@ -167,7 +169,8 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU // REMOTE ACCOUNT REQUEST WITHOUT KEY CACHED LOCALLY // the request is remote and we don't have the public key yet, // so we need to authenticate the request properly by dereferencing the remote key - transport, err := f.GetTransportForUser(requestedUsername) + l.Tracef("proceeding with dereference for uncached public key %s", requestingPublicKeyID) + transport, err := f.transportController.NewTransportForUsername(requestedUsername) if err != nil { return nil, false, fmt.Errorf("transport err: %s", err) } @@ -209,15 +212,28 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU } pkOwnerURI = pkOwnerProp.GetIRI() } + + // after all that, public key should be defined if publicKey == nil { return nil, false, errors.New("returned public key was empty") } // do the actual authentication here! - algo := httpsig.RSA_SHA256 // TODO: make this more robust - if err := verifier.Verify(publicKey, algo); err != nil { - return nil, false, nil + algos := []httpsig.Algorithm{ + httpsig.RSA_SHA512, + httpsig.RSA_SHA256, + httpsig.ED25519, } - return pkOwnerURI, true, nil + for _, algo := range algos { + l.Tracef("trying algo: %s", algo) + if err := verifier.Verify(publicKey, algo); err == nil { + l.Tracef("authentication for %s PASSED with algorithm %s", pkOwnerURI, algo) + return pkOwnerURI, true, nil + } + l.Tracef("authentication for %s NOT PASSED with algorithm %s: %s", pkOwnerURI, algo, err) + } + + l.Infof("authentication not passed for %s", pkOwnerURI) + return nil, false, nil } diff --git a/internal/federation/dereference.go b/internal/federation/dereference.go index b87462acd..8975d6c0c 100644 --- a/internal/federation/dereference.go +++ b/internal/federation/dereference.go @@ -1,526 +1,32 @@ package federation import ( - "context" - "encoding/json" - "errors" - "fmt" "net/url" - "github.com/go-fed/activity/streams" - "github.com/go-fed/activity/streams/vocab" - "github.com/sirupsen/logrus" - "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/transport" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) -func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) { - f.startHandshake(username, remoteAccountID) - defer f.stopHandshake(username, remoteAccountID) - - if blocked, err := f.blockedDomain(remoteAccountID.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceRemoteAccount: domain %s is blocked", remoteAccountID.Host) - } - - transport, err := f.GetTransportForUser(username) - if err != nil { - return nil, fmt.Errorf("transport err: %s", err) - } - - b, err := transport.Dereference(context.Background(), remoteAccountID) - if err != nil { - return nil, fmt.Errorf("error deferencing %s: %s", remoteAccountID.String(), err) - } - - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err) - } - - t, err := streams.ToType(context.Background(), m) - if err != nil { - return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err) - } - - switch t.GetTypeName() { - case string(gtsmodel.ActivityStreamsPerson): - p, ok := t.(vocab.ActivityStreamsPerson) - if !ok { - return nil, errors.New("error resolving type as activitystreams person") - } - return p, nil - case string(gtsmodel.ActivityStreamsApplication): - p, ok := t.(vocab.ActivityStreamsApplication) - if !ok { - return nil, errors.New("error resolving type as activitystreams application") - } - return p, nil - case string(gtsmodel.ActivityStreamsService): - p, ok := t.(vocab.ActivityStreamsService) - if !ok { - return nil, errors.New("error resolving type as activitystreams service") - } - return p, nil - } - - return nil, fmt.Errorf("type name %s not supported", t.GetTypeName()) +func (f *federator) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) { + return f.dereferencer.GetRemoteAccount(username, remoteAccountID, refresh) } -func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) { - if blocked, err := f.blockedDomain(remoteStatusID.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceRemoteStatus: domain %s is blocked", remoteStatusID.Host) - } - - transport, err := f.GetTransportForUser(username) - if err != nil { - return nil, fmt.Errorf("transport err: %s", err) - } - - b, err := transport.Dereference(context.Background(), remoteStatusID) - if err != nil { - return nil, fmt.Errorf("error deferencing %s: %s", remoteStatusID.String(), err) - } - - m := make(map[string]interface{}) - if err := json.Unmarshal(b, &m); err != nil { - return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err) - } - - t, err := streams.ToType(context.Background(), m) - if err != nil { - return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err) - } - - // Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile - switch t.GetTypeName() { - case gtsmodel.ActivityStreamsArticle: - p, ok := t.(vocab.ActivityStreamsArticle) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsArticle") - } - return p, nil - case gtsmodel.ActivityStreamsDocument: - p, ok := t.(vocab.ActivityStreamsDocument) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsDocument") - } - return p, nil - case gtsmodel.ActivityStreamsImage: - p, ok := t.(vocab.ActivityStreamsImage) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsImage") - } - return p, nil - case gtsmodel.ActivityStreamsVideo: - p, ok := t.(vocab.ActivityStreamsVideo) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsVideo") - } - return p, nil - case gtsmodel.ActivityStreamsNote: - p, ok := t.(vocab.ActivityStreamsNote) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsNote") - } - return p, nil - case gtsmodel.ActivityStreamsPage: - p, ok := t.(vocab.ActivityStreamsPage) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsPage") - } - return p, nil - case gtsmodel.ActivityStreamsEvent: - p, ok := t.(vocab.ActivityStreamsEvent) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsEvent") - } - return p, nil - case gtsmodel.ActivityStreamsPlace: - p, ok := t.(vocab.ActivityStreamsPlace) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsPlace") - } - return p, nil - case gtsmodel.ActivityStreamsProfile: - p, ok := t.(vocab.ActivityStreamsProfile) - if !ok { - return nil, errors.New("error resolving type as ActivityStreamsProfile") - } - return p, nil - } - - return nil, fmt.Errorf("type name %s not supported", t.GetTypeName()) +func (f *federator) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) { + return f.dereferencer.GetRemoteStatus(username, remoteStatusID, refresh) } -func (f *federator) DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { - if blocked, err := f.blockedDomain(remoteInstanceURI.Host); blocked || err != nil { - return nil, fmt.Errorf("DereferenceRemoteInstance: domain %s is blocked", remoteInstanceURI.Host) - } - - transport, err := f.GetTransportForUser(username) - if err != nil { - return nil, fmt.Errorf("transport err: %s", err) - } - - return transport.DereferenceInstance(context.Background(), remoteInstanceURI) +func (f *federator) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) { + return f.dereferencer.EnrichRemoteStatus(username, status) } -// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming -// federated status, back in the federating db's Create function. -// -// When a status comes in from the federation API, there are certain fields that -// haven't been dereferenced yet, because we needed to provide a snappy synchronous -// response to the caller. By the time it reaches this function though, it's being -// processed asynchronously, so we have all the time in the world to fetch the various -// bits and bobs that are attached to the status, and properly flesh it out, before we -// send the status to any timelines and notify people. -// -// Things to dereference and fetch here: -// -// 1. Media attachments. -// 2. Hashtags. -// 3. Emojis. -// 4. Mentions. -// 5. Posting account. -// 6. Replied-to-status. -// -// SIDE EFFECTS: -// This function will deference all of the above, insert them in the database as necessary, -// and attach them to the status. The status itself will not be added to the database yet, -// that's up the caller to do. -func (f *federator) DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error { - l := f.log.WithFields(logrus.Fields{ - "func": "dereferenceStatusFields", - "status": fmt.Sprintf("%+v", status), - }) - l.Debug("entering function") - - statusURI, err := url.Parse(status.URI) - if err != nil { - return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err) - } - if blocked, err := f.blockedDomain(statusURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host) - } - - t, err := f.GetTransportForUser(requestingUsername) - if err != nil { - return fmt.Errorf("error creating transport: %s", err) - } - - // the status should have an ID by now, but just in case it doesn't let's generate one here - // because we'll need it further down - if status.ID == "" { - newID, err := id.NewULIDFromTime(status.CreatedAt) - if err != nil { - return err - } - status.ID = newID - } - - // 1. Media attachments. - // - // At this point we should know: - // * the media type of the file we're looking for (a.File.ContentType) - // * the blurhash (a.Blurhash) - // * the file type (a.Type) - // * the remote URL (a.RemoteURL) - // This should be enough to pass along to the media processor. - attachmentIDs := []string{} - for _, a := range status.GTSMediaAttachments { - l.Debugf("dereferencing attachment: %+v", a) - - // it might have been processed elsewhere so check first if it's already in the database or not - maybeAttachment := >smodel.MediaAttachment{} - err := f.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment) - if err == nil { - // we already have it in the db, dereferenced, no need to do it again - l.Debugf("attachment already exists with id %s", maybeAttachment.ID) - attachmentIDs = append(attachmentIDs, maybeAttachment.ID) - continue - } - if _, ok := err.(db.ErrNoEntries); !ok { - // we have a real error - return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err) - } - // it just doesn't exist yet so carry on - l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a) - deferencedAttachment, err := f.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID) - if err != nil { - l.Errorf("error dereferencing status attachment: %s", err) - continue - } - l.Debugf("dereferenced attachment: %+v", deferencedAttachment) - deferencedAttachment.StatusID = status.ID - deferencedAttachment.Description = a.Description - if err := f.db.Put(deferencedAttachment); err != nil { - return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err) - } - attachmentIDs = append(attachmentIDs, deferencedAttachment.ID) - } - status.Attachments = attachmentIDs - - // 2. Hashtags - - // 3. Emojis - - // 4. Mentions - // At this point, mentions should have the namestring and mentionedAccountURI set on them. - // - // We should dereference any accounts mentioned here which we don't have in our db yet, by their URI. - mentions := []string{} - for _, m := range status.GTSMentions { - if m.ID == "" { - mID, err := id.NewRandomULID() - if err != nil { - return err - } - m.ID = mID - } - - uri, err := url.Parse(m.MentionedAccountURI) - if err != nil { - l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err) - continue - } - - m.StatusID = status.ID - m.OriginAccountID = status.GTSAuthorAccount.ID - m.OriginAccountURI = status.GTSAuthorAccount.URI - - targetAccount := >smodel.Account{} - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil { - // proper error - if _, ok := err.(db.ErrNoEntries); !ok { - return fmt.Errorf("db error checking for account with uri %s", uri.String()) - } - - // we just don't have it yet, so we should go get it.... - accountable, err := f.DereferenceRemoteAccount(requestingUsername, uri) - if err != nil { - // we can't dereference it so just skip it - l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err) - continue - } - - targetAccount, err = f.typeConverter.ASRepresentationToAccount(accountable, false) - if err != nil { - l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err) - continue - } - - targetAccountID, err := id.NewRandomULID() - if err != nil { - return err - } - targetAccount.ID = targetAccountID - - if err := f.db.Put(targetAccount); err != nil { - return fmt.Errorf("db error inserting account with uri %s", uri.String()) - } - } - - // by this point, we know the targetAccount exists in our database with an ID :) - m.TargetAccountID = targetAccount.ID - if err := f.db.Put(m); err != nil { - return fmt.Errorf("error creating mention: %s", err) - } - mentions = append(mentions, m.ID) - } - status.Mentions = mentions - - return nil +func (f *federator) DereferenceRemoteThread(username string, statusIRI *url.URL) error { + return f.dereferencer.DereferenceThread(username, statusIRI) } -func (f *federator) DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error { - l := f.log.WithFields(logrus.Fields{ - "func": "dereferenceAccountFields", - "requestingUsername": requestingUsername, - }) - - accountURI, err := url.Parse(account.URI) - if err != nil { - return fmt.Errorf("DereferenceAccountFields: couldn't parse account URI %s: %s", account.URI, err) - } - if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceAccountFields: domain %s is blocked", accountURI.Host) - } - - t, err := f.GetTransportForUser(requestingUsername) - if err != nil { - return fmt.Errorf("error getting transport for user: %s", err) - } - - // fetch the header and avatar - if err := f.fetchHeaderAndAviForAccount(account, t, refresh); err != nil { - // if this doesn't work, just skip it -- we can do it later - l.Debugf("error fetching header/avi for account: %s", err) - } - - if err := f.db.UpdateByID(account.ID, account); err != nil { - return fmt.Errorf("error updating account in database: %s", err) - } - - return nil +func (f *federator) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { + return f.dereferencer.GetRemoteInstance(username, remoteInstanceURI) } func (f *federator) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error { - if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" { - // we can't do anything unfortunately - return errors.New("DereferenceAnnounce: no URI to dereference") - } - - boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI) - if err != nil { - return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err) - } - if blocked, err := f.blockedDomain(boostedStatusURI.Host); blocked || err != nil { - return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host) - } - - // check if we already have the boosted status in the database - boostedStatus := >smodel.Status{} - err = f.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus) - if err == nil { - // nice, we already have it so we don't actually need to dereference it from remote - announce.Content = boostedStatus.Content - announce.ContentWarning = boostedStatus.ContentWarning - announce.ActivityStreamsType = boostedStatus.ActivityStreamsType - announce.Sensitive = boostedStatus.Sensitive - announce.Language = boostedStatus.Language - announce.Text = boostedStatus.Text - announce.BoostOfID = boostedStatus.ID - announce.BoostOfAccountID = boostedStatus.AccountID - announce.Visibility = boostedStatus.Visibility - announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced - announce.GTSBoostedStatus = boostedStatus - return nil - } - - // we don't have it so we need to dereference it - statusable, err := f.DereferenceRemoteStatus(requestingUsername, boostedStatusURI) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err) - } - - // make sure we have the author account in the db - attributedToProp := statusable.GetActivityStreamsAttributedTo() - for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() { - accountURI := iter.GetIRI() - if accountURI == nil { - continue - } - - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, >smodel.Account{}); err == nil { - // we already have it, fine - continue - } - - // we don't have the boosted status author account yet so dereference it - accountable, err := f.DereferenceRemoteAccount(requestingUsername, accountURI) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err) - } - account, err := f.typeConverter.ASRepresentationToAccount(accountable, false) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err) - } - - accountID, err := id.NewRandomULID() - if err != nil { - return err - } - account.ID = accountID - - if err := f.db.Put(account); err != nil { - return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err) - } - - if err := f.DereferenceAccountFields(account, requestingUsername, false); err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err) - } - } - - // now convert the statusable into something we can understand - boostedStatus, err = f.typeConverter.ASStatusToStatus(statusable) - if err != nil { - return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err) - } - - boostedStatusID, err := id.NewULIDFromTime(boostedStatus.CreatedAt) - if err != nil { - return nil - } - boostedStatus.ID = boostedStatusID - - if err := f.db.Put(boostedStatus); err != nil { - return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err) - } - - // now dereference additional fields straight away (we're already async here so we have time) - if err := f.DereferenceStatusFields(boostedStatus, requestingUsername); err != nil { - return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err) - } - - // update with the newly dereferenced fields - if err := f.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil { - return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err) - } - - // we have everything we need! - announce.Content = boostedStatus.Content - announce.ContentWarning = boostedStatus.ContentWarning - announce.ActivityStreamsType = boostedStatus.ActivityStreamsType - announce.Sensitive = boostedStatus.Sensitive - announce.Language = boostedStatus.Language - announce.Text = boostedStatus.Text - announce.BoostOfID = boostedStatus.ID - announce.BoostOfAccountID = boostedStatus.AccountID - announce.Visibility = boostedStatus.Visibility - announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced - announce.GTSBoostedStatus = boostedStatus - return nil -} - -// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport -// on behalf of requestingUsername. -// -// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary. -// -// SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated -// to reflect the creation of these new attachments. -func (f *federator) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error { - accountURI, err := url.Parse(targetAccount.URI) - if err != nil { - return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err) - } - if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil { - return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host) - } - - if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { - a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.AvatarRemoteURL, - Avatar: true, - }, targetAccount.ID) - if err != nil { - return fmt.Errorf("error processing avatar for user: %s", err) - } - targetAccount.AvatarMediaAttachmentID = a.ID - } - - if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { - a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ - RemoteURL: targetAccount.HeaderRemoteURL, - Header: true, - }, targetAccount.ID) - if err != nil { - return fmt.Errorf("error processing header for user: %s", err) - } - targetAccount.HeaderMediaAttachmentID = a.ID - } - return nil + return f.dereferencer.DereferenceAnnounce(announce, requestingUsername) } diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go new file mode 100644 index 000000000..c403ec66f --- /dev/null +++ b/internal/federation/dereferencing/account.go @@ -0,0 +1,243 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/transport" +) + +// EnrichRemoteAccount takes an account that's already been inserted into the database in a minimal form, +// and populates it with additional fields, media, etc. +// +// EnrichRemoteAccount is mostly useful for calling after an account has been initially created by +// the federatingDB's Create function, or during the federated authorization flow. +func (d *deref) EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error) { + if err := d.populateAccountFields(account, username, false); err != nil { + return nil, err + } + + if err := d.db.UpdateByID(account.ID, account); err != nil { + return nil, fmt.Errorf("EnrichRemoteAccount: error updating account: %s", err) + } + + return account, nil +} + +// GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account, +// puts it in the database, and returns it to a caller. The boolean indicates whether the account is new +// to us or not. If we haven't seen the account before, bool will be true. If we have seen the account before, +// it will be false. +// +// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling +// the remote instance again. +// +// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true). +func (d *deref) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) { + new := true + + // check if we already have the account in our db + maybeAccount := >smodel.Account{} + if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteAccountID.String()}}, maybeAccount); err == nil { + // we've seen this account before so it's not new + new = false + + // if we're not being asked to refresh, we can just return the maybeAccount as-is and avoid doing any external calls + if !refresh { + return maybeAccount, new, nil + } + } + + accountable, err := d.dereferenceAccountable(username, remoteAccountID) + if err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error dereferencing accountable: %s", err) + } + + gtsAccount, err := d.typeConverter.ASRepresentationToAccount(accountable, false) + if err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error converting accountable to account: %s", err) + } + + if new { + // generate a new id since we haven't seen this account before, and do a put + ulid, err := id.NewRandomULID() + if err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error generating new id for account: %s", err) + } + gtsAccount.ID = ulid + + if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) + } + + if err := d.db.Put(gtsAccount); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error putting new account: %s", err) + } + } else { + // take the id we already have and do an update + gtsAccount.ID = maybeAccount.ID + + if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err) + } + + if err := d.db.UpdateByID(gtsAccount.ID, gtsAccount); err != nil { + return nil, new, fmt.Errorf("FullyDereferenceAccount: error updating existing account: %s", err) + } + } + + return gtsAccount, new, nil +} + +// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever +// it finds as something that an account model can be constructed out of. +// +// Will work for Person, Application, or Service models. +func (d *deref) dereferenceAccountable(username string, remoteAccountID *url.URL) (ap.Accountable, error) { + d.startHandshake(username, remoteAccountID) + defer d.stopHandshake(username, remoteAccountID) + + if blocked, err := d.blockedDomain(remoteAccountID.Host); blocked || err != nil { + return nil, fmt.Errorf("DereferenceAccountable: domain %s is blocked", remoteAccountID.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("DereferenceAccountable: transport err: %s", err) + } + + b, err := transport.Dereference(context.Background(), remoteAccountID) + if err != nil { + return nil, fmt.Errorf("DereferenceAccountable: error deferencing %s: %s", remoteAccountID.String(), err) + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("DereferenceAccountable: error unmarshalling bytes into json: %s", err) + } + + t, err := streams.ToType(context.Background(), m) + if err != nil { + return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %s", err) + } + + switch t.GetTypeName() { + case string(gtsmodel.ActivityStreamsPerson): + p, ok := t.(vocab.ActivityStreamsPerson) + if !ok { + return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams person") + } + return p, nil + case string(gtsmodel.ActivityStreamsApplication): + p, ok := t.(vocab.ActivityStreamsApplication) + if !ok { + return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams application") + } + return p, nil + case string(gtsmodel.ActivityStreamsService): + p, ok := t.(vocab.ActivityStreamsService) + if !ok { + return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams service") + } + return p, nil + } + + return nil, fmt.Errorf("DereferenceAccountable: type name %s not supported", t.GetTypeName()) +} + +// populateAccountFields populates any fields on the given account that weren't populated by the initial +// dereferencing. This includes things like header and avatar etc. +func (d *deref) populateAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error { + l := d.log.WithFields(logrus.Fields{ + "func": "PopulateAccountFields", + "requestingUsername": requestingUsername, + }) + + accountURI, err := url.Parse(account.URI) + if err != nil { + return fmt.Errorf("PopulateAccountFields: couldn't parse account URI %s: %s", account.URI, err) + } + if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil { + return fmt.Errorf("PopulateAccountFields: domain %s is blocked", accountURI.Host) + } + + t, err := d.transportController.NewTransportForUsername(requestingUsername) + if err != nil { + return fmt.Errorf("PopulateAccountFields: error getting transport for user: %s", err) + } + + // fetch the header and avatar + if err := d.fetchHeaderAndAviForAccount(account, t, refresh); err != nil { + // if this doesn't work, just skip it -- we can do it later + l.Debugf("error fetching header/avi for account: %s", err) + } + + return nil +} + +// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport +// on behalf of requestingUsername. +// +// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary. +// +// SIDE EFFECTS: remote header and avatar will be stored in local storage. +func (d *deref) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error { + accountURI, err := url.Parse(targetAccount.URI) + if err != nil { + return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err) + } + if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil { + return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host) + } + + if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { + a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ + RemoteURL: targetAccount.AvatarRemoteURL, + Avatar: true, + }, targetAccount.ID) + if err != nil { + return fmt.Errorf("error processing avatar for user: %s", err) + } + targetAccount.AvatarMediaAttachmentID = a.ID + } + + if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { + a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, >smodel.MediaAttachment{ + RemoteURL: targetAccount.HeaderRemoteURL, + Header: true, + }, targetAccount.ID) + if err != nil { + return fmt.Errorf("error processing header for user: %s", err) + } + targetAccount.HeaderMediaAttachmentID = a.ID + } + return nil +} diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go new file mode 100644 index 000000000..2522a4034 --- /dev/null +++ b/internal/federation/dereferencing/announce.go @@ -0,0 +1,65 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "errors" + "fmt" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error { + if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" { + // we can't do anything unfortunately + return errors.New("DereferenceAnnounce: no URI to dereference") + } + + boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI) + if err != nil { + return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err) + } + if blocked, err := d.blockedDomain(boostedStatusURI.Host); blocked || err != nil { + return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host) + } + + // dereference statuses in the thread of the boosted status + if err := d.DereferenceThread(requestingUsername, boostedStatusURI); err != nil { + return fmt.Errorf("DereferenceAnnounce: error dereferencing thread of boosted status: %s", err) + } + + boostedStatus, _, _, err := d.GetRemoteStatus(requestingUsername, boostedStatusURI, false) + if err != nil { + return fmt.Errorf("DereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err) + } + + announce.Content = boostedStatus.Content + announce.ContentWarning = boostedStatus.ContentWarning + announce.ActivityStreamsType = boostedStatus.ActivityStreamsType + announce.Sensitive = boostedStatus.Sensitive + announce.Language = boostedStatus.Language + announce.Text = boostedStatus.Text + announce.BoostOfID = boostedStatus.ID + announce.BoostOfAccountID = boostedStatus.AccountID + announce.Visibility = boostedStatus.Visibility + announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced + announce.GTSBoostedStatus = boostedStatus + return nil +} diff --git a/internal/federation/dereferencing/blocked.go b/internal/federation/dereferencing/blocked.go new file mode 100644 index 000000000..a66afbb60 --- /dev/null +++ b/internal/federation/dereferencing/blocked.go @@ -0,0 +1,41 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) blockedDomain(host string) (bool, error) { + b := >smodel.DomainBlock{} + err := d.db.GetWhere([]db.Where{{Key: "domain", Value: host, CaseInsensitive: true}}, b) + if err == nil { + // block exists + return true, nil + } + + if _, ok := err.(db.ErrNoEntries); ok { + // there are no entries so there's no block + return false, nil + } + + // there's an actual error + return false, err +} diff --git a/internal/federation/dereferencing/collectionpage.go b/internal/federation/dereferencing/collectionpage.go new file mode 100644 index 000000000..5feadc1ad --- /dev/null +++ b/internal/federation/dereferencing/collectionpage.go @@ -0,0 +1,70 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// DereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong. +func (d *deref) DereferenceCollectionPage(username string, pageIRI *url.URL) (ap.CollectionPageable, error) { + if blocked, err := d.blockedDomain(pageIRI.Host); blocked || err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: domain %s is blocked", pageIRI.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error creating transport: %s", err) + } + + b, err := transport.Dereference(context.Background(), pageIRI) + if err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error deferencing %s: %s", pageIRI.String(), err) + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error unmarshalling bytes into json: %s", err) + } + + t, err := streams.ToType(context.Background(), m) + if err != nil { + return nil, fmt.Errorf("DereferenceCollectionPage: error resolving json into ap vocab type: %s", err) + } + + if t.GetTypeName() != gtsmodel.ActivityStreamsCollectionPage { + return nil, fmt.Errorf("DereferenceCollectionPage: type name %s not supported", t.GetTypeName()) + } + + p, ok := t.(vocab.ActivityStreamsCollectionPage) + if !ok { + return nil, errors.New("DereferenceCollectionPage: error resolving type as activitystreams collection page") + } + + return p, nil +} diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go new file mode 100644 index 000000000..03b90569a --- /dev/null +++ b/internal/federation/dereferencing/dereferencer.go @@ -0,0 +1,73 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "net/url" + "sync" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/typeutils" +) + +// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances. +type Dereferencer interface { + GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) + EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error) + + GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) + EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) + + GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) + + DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error + DereferenceThread(username string, statusIRI *url.URL) error + + Handshaking(username string, remoteAccountID *url.URL) bool +} + +type deref struct { + log *logrus.Logger + db db.DB + typeConverter typeutils.TypeConverter + transportController transport.Controller + mediaHandler media.Handler + config *config.Config + handshakes map[string][]*url.URL + handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map +} + +// NewDereferencer returns a Dereferencer initialized with the given parameters. +func NewDereferencer(config *config.Config, db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaHandler media.Handler, log *logrus.Logger) Dereferencer { + return &deref{ + log: log, + db: db, + typeConverter: typeConverter, + transportController: transportController, + mediaHandler: mediaHandler, + config: config, + handshakeSync: &sync.Mutex{}, + } +} diff --git a/internal/federation/dereferencing/handshake.go b/internal/federation/dereferencing/handshake.go new file mode 100644 index 000000000..cda8eafd0 --- /dev/null +++ b/internal/federation/dereferencing/handshake.go @@ -0,0 +1,98 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import "net/url" + +func (d *deref) Handshaking(username string, remoteAccountID *url.URL) bool { + d.handshakeSync.Lock() + defer d.handshakeSync.Unlock() + + if d.handshakes == nil { + // handshakes isn't even initialized yet so we can't be handshaking with anyone + return false + } + + remoteIDs, ok := d.handshakes[username] + if !ok { + // user isn't handshaking with anyone, bail + return false + } + + for _, id := range remoteIDs { + if id.String() == remoteAccountID.String() { + // we are currently handshaking with the remote account, yep + return true + } + } + + // didn't find it which means we're not handshaking + return false +} + +func (d *deref) startHandshake(username string, remoteAccountID *url.URL) { + d.handshakeSync.Lock() + defer d.handshakeSync.Unlock() + + // lazily initialize handshakes + if d.handshakes == nil { + d.handshakes = make(map[string][]*url.URL) + } + + remoteIDs, ok := d.handshakes[username] + if !ok { + // there was nothing in there yet, so just add this entry and return + d.handshakes[username] = []*url.URL{remoteAccountID} + return + } + + // add the remote ID to the slice + remoteIDs = append(remoteIDs, remoteAccountID) + d.handshakes[username] = remoteIDs +} + +func (d *deref) stopHandshake(username string, remoteAccountID *url.URL) { + d.handshakeSync.Lock() + defer d.handshakeSync.Unlock() + + if d.handshakes == nil { + return + } + + remoteIDs, ok := d.handshakes[username] + if !ok { + // there was nothing in there yet anyway so just bail + return + } + + newRemoteIDs := []*url.URL{} + for _, id := range remoteIDs { + if id.String() != remoteAccountID.String() { + newRemoteIDs = append(newRemoteIDs, id) + } + } + + if len(newRemoteIDs) == 0 { + // there are no handshakes so just remove this user entry from the map and save a few bytes + delete(d.handshakes, username) + } else { + // there are still other handshakes ongoing + d.handshakes[username] = newRemoteIDs + } +} diff --git a/internal/federation/dereferencing/instance.go b/internal/federation/dereferencing/instance.go new file mode 100644 index 000000000..80f626662 --- /dev/null +++ b/internal/federation/dereferencing/instance.go @@ -0,0 +1,40 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "fmt" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (d *deref) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) { + if blocked, err := d.blockedDomain(remoteInstanceURI.Host); blocked || err != nil { + return nil, fmt.Errorf("GetRemoteInstance: domain %s is blocked", remoteInstanceURI.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("transport err: %s", err) + } + + return transport.DereferenceInstance(context.Background(), remoteInstanceURI) +} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go new file mode 100644 index 000000000..b05f6e72c --- /dev/null +++ b/internal/federation/dereferencing/status.go @@ -0,0 +1,369 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/go-fed/activity/streams" + "github.com/go-fed/activity/streams/vocab" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/id" +) + +// EnrichRemoteStatus takes a status that's already been inserted into the database in a minimal form, +// and populates it with additional fields, media, etc. +// +// EnrichRemoteStatus is mostly useful for calling after a status has been initially created by +// the federatingDB's Create function, but additional dereferencing is needed on it. +func (d *deref) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) { + if err := d.populateStatusFields(status, username); err != nil { + return nil, err + } + + if err := d.db.UpdateByID(status.ID, status); err != nil { + return nil, fmt.Errorf("EnrichRemoteStatus: error updating status: %s", err) + } + + return status, nil +} + +// GetRemoteStatus completely dereferences a remote status, converts it to a GtS model status, +// puts it in the database, and returns it to a caller. The boolean indicates whether the status is new +// to us or not. If we haven't seen the status before, bool will be true. If we have seen the status before, +// it will be false. +// +// If refresh is true, then even if we have the status in our database already, it will be dereferenced from its +// remote representation, as will its owner. +// +// If a dereference was performed, then the function also returns the ap.Statusable representation for further processing. +// +// SIDE EFFECTS: remote status will be stored in the database, and the remote status owner will also be stored. +func (d *deref) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) { + new := true + + // check if we already have the status in our db + maybeStatus := >smodel.Status{} + if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteStatusID.String()}}, maybeStatus); err == nil { + // we've seen this status before so it's not new + new = false + + // if we're not being asked to refresh, we can just return the maybeStatus as-is and avoid doing any external calls + if !refresh { + return maybeStatus, nil, new, nil + } + } + + statusable, err := d.dereferenceStatusable(username, remoteStatusID) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error dereferencing statusable: %s", err) + } + + accountURI, err := ap.ExtractAttributedTo(statusable) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %s", err) + } + + // do this so we know we have the remote account of the status in the db + _, _, err = d.GetRemoteAccount(username, accountURI, false) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: couldn't derive status author: %s", err) + } + + gtsStatus, err := d.typeConverter.ASStatusToStatus(statusable) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error converting statusable to status: %s", err) + } + + if new { + ulid, err := id.NewULIDFromTime(gtsStatus.CreatedAt) + if err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error generating new id for status: %s", err) + } + gtsStatus.ID = ulid + + if err := d.populateStatusFields(gtsStatus, username); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err) + } + + if err := d.db.Put(gtsStatus); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error putting new status: %s", err) + } + } else { + gtsStatus.ID = maybeStatus.ID + + if err := d.populateStatusFields(gtsStatus, username); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err) + } + + if err := d.db.UpdateByID(gtsStatus.ID, gtsStatus); err != nil { + return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error updating status: %s", err) + } + } + + return gtsStatus, statusable, new, nil +} + +func (d *deref) dereferenceStatusable(username string, remoteStatusID *url.URL) (ap.Statusable, error) { + if blocked, err := d.blockedDomain(remoteStatusID.Host); blocked || err != nil { + return nil, fmt.Errorf("DereferenceStatusable: domain %s is blocked", remoteStatusID.Host) + } + + transport, err := d.transportController.NewTransportForUsername(username) + if err != nil { + return nil, fmt.Errorf("DereferenceStatusable: transport err: %s", err) + } + + b, err := transport.Dereference(context.Background(), remoteStatusID) + if err != nil { + return nil, fmt.Errorf("DereferenceStatusable: error deferencing %s: %s", remoteStatusID.String(), err) + } + + m := make(map[string]interface{}) + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("DereferenceStatusable: error unmarshalling bytes into json: %s", err) + } + + t, err := streams.ToType(context.Background(), m) + if err != nil { + return nil, fmt.Errorf("DereferenceStatusable: error resolving json into ap vocab type: %s", err) + } + + // Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile + switch t.GetTypeName() { + case gtsmodel.ActivityStreamsArticle: + p, ok := t.(vocab.ActivityStreamsArticle) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsArticle") + } + return p, nil + case gtsmodel.ActivityStreamsDocument: + p, ok := t.(vocab.ActivityStreamsDocument) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsDocument") + } + return p, nil + case gtsmodel.ActivityStreamsImage: + p, ok := t.(vocab.ActivityStreamsImage) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsImage") + } + return p, nil + case gtsmodel.ActivityStreamsVideo: + p, ok := t.(vocab.ActivityStreamsVideo) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsVideo") + } + return p, nil + case gtsmodel.ActivityStreamsNote: + p, ok := t.(vocab.ActivityStreamsNote) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsNote") + } + return p, nil + case gtsmodel.ActivityStreamsPage: + p, ok := t.(vocab.ActivityStreamsPage) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPage") + } + return p, nil + case gtsmodel.ActivityStreamsEvent: + p, ok := t.(vocab.ActivityStreamsEvent) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsEvent") + } + return p, nil + case gtsmodel.ActivityStreamsPlace: + p, ok := t.(vocab.ActivityStreamsPlace) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPlace") + } + return p, nil + case gtsmodel.ActivityStreamsProfile: + p, ok := t.(vocab.ActivityStreamsProfile) + if !ok { + return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsProfile") + } + return p, nil + } + + return nil, fmt.Errorf("DereferenceStatusable: type name %s not supported", t.GetTypeName()) +} + +// populateStatusFields fetches all the information we temporarily pinned to an incoming +// federated status, back in the federating db's Create function. +// +// When a status comes in from the federation API, there are certain fields that +// haven't been dereferenced yet, because we needed to provide a snappy synchronous +// response to the caller. By the time it reaches this function though, it's being +// processed asynchronously, so we have all the time in the world to fetch the various +// bits and bobs that are attached to the status, and properly flesh it out, before we +// send the status to any timelines and notify people. +// +// Things to dereference and fetch here: +// +// 1. Media attachments. +// 2. Hashtags. +// 3. Emojis. +// 4. Mentions. +// 5. Posting account. +// 6. Replied-to-status. +// +// SIDE EFFECTS: +// This function will deference all of the above, insert them in the database as necessary, +// and attach them to the status. The status itself will not be added to the database yet, +// that's up the caller to do. +func (d *deref) populateStatusFields(status *gtsmodel.Status, requestingUsername string) error { + l := d.log.WithFields(logrus.Fields{ + "func": "dereferenceStatusFields", + "status": fmt.Sprintf("%+v", status), + }) + l.Debug("entering function") + + // make sure we have a status URI and that the domain in question isn't blocked + statusURI, err := url.Parse(status.URI) + if err != nil { + return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err) + } + if blocked, err := d.blockedDomain(statusURI.Host); blocked || err != nil { + return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host) + } + + // we can continue -- create a new transport here because we'll probably need it + t, err := d.transportController.NewTransportForUsername(requestingUsername) + if err != nil { + return fmt.Errorf("error creating transport: %s", err) + } + + // in case the status doesn't have an id yet (ie., it hasn't entered the database yet), then create one + if status.ID == "" { + newID, err := id.NewULIDFromTime(status.CreatedAt) + if err != nil { + return err + } + status.ID = newID + } + + // 1. Media attachments. + // + // At this point we should know: + // * the media type of the file we're looking for (a.File.ContentType) + // * the blurhash (a.Blurhash) + // * the file type (a.Type) + // * the remote URL (a.RemoteURL) + // This should be enough to pass along to the media processor. + attachmentIDs := []string{} + for _, a := range status.GTSMediaAttachments { + l.Tracef("dereferencing attachment: %+v", a) + + // it might have been processed elsewhere so check first if it's already in the database or not + maybeAttachment := >smodel.MediaAttachment{} + err := d.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment) + if err == nil { + // we already have it in the db, dereferenced, no need to do it again + l.Tracef("attachment already exists with id %s", maybeAttachment.ID) + attachmentIDs = append(attachmentIDs, maybeAttachment.ID) + continue + } + if _, ok := err.(db.ErrNoEntries); !ok { + // we have a real error + return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err) + } + // it just doesn't exist yet so carry on + l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a) + deferencedAttachment, err := d.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID) + if err != nil { + l.Errorf("error dereferencing status attachment: %s", err) + continue + } + l.Debugf("dereferenced attachment: %+v", deferencedAttachment) + deferencedAttachment.StatusID = status.ID + deferencedAttachment.Description = a.Description + if err := d.db.Put(deferencedAttachment); err != nil { + return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err) + } + attachmentIDs = append(attachmentIDs, deferencedAttachment.ID) + } + status.Attachments = attachmentIDs + + // 2. Hashtags + + // 3. Emojis + + // 4. Mentions + // At this point, mentions should have the namestring and mentionedAccountURI set on them. + // + // We should dereference any accounts mentioned here which we don't have in our db yet, by their URI. + mentions := []string{} + for _, m := range status.GTSMentions { + + if m.ID != "" { + continue + // we've already populated this mention, since it has an ID + } + + mID, err := id.NewRandomULID() + if err != nil { + return err + } + m.ID = mID + + uri, err := url.Parse(m.MentionedAccountURI) + if err != nil { + l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err) + continue + } + + m.StatusID = status.ID + m.OriginAccountID = status.GTSAuthorAccount.ID + m.OriginAccountURI = status.GTSAuthorAccount.URI + + targetAccount, _, err := d.GetRemoteAccount(requestingUsername, uri, false) + if err != nil { + continue + } + + // by this point, we know the targetAccount exists in our database with an ID :) + m.TargetAccountID = targetAccount.ID + if err := d.db.Put(m); err != nil { + return fmt.Errorf("error creating mention: %s", err) + } + mentions = append(mentions, m.ID) + } + status.Mentions = mentions + + // status has replyToURI but we don't have an ID yet for the status it replies to + if status.InReplyToURI != "" && status.InReplyToID == "" { + replyToStatus := >smodel.Status{} + if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: status.InReplyToURI}}, replyToStatus); err == nil { + // we have the status + status.InReplyToID = replyToStatus.ID + status.InReplyToAccountID = replyToStatus.AccountID + } + } + + return nil +} diff --git a/internal/federation/dereferencing/thread.go b/internal/federation/dereferencing/thread.go new file mode 100644 index 000000000..2a407f923 --- /dev/null +++ b/internal/federation/dereferencing/thread.go @@ -0,0 +1,250 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package dereferencing + +import ( + "fmt" + "net/url" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +// DereferenceThread takes a statusable (something that has withReplies and withInReplyTo), +// and dereferences statusables in the conversation. +// +// This process involves working up and down the chain of replies, and parsing through the collections of IDs +// presented by remote instances as part of their replies collections, and will likely involve making several calls to +// multiple different hosts. +func (d *deref) DereferenceThread(username string, statusIRI *url.URL) error { + l := d.log.WithFields(logrus.Fields{ + "func": "DereferenceThread", + "username": username, + "statusIRI": statusIRI.String(), + }) + l.Debug("entering DereferenceThread") + + // if it's our status we already have everything stashed so we can bail early + if statusIRI.Host == d.config.Host { + l.Debug("iri belongs to us, bailing") + return nil + } + + // first make sure we have this status in our db + _, statusable, _, err := d.GetRemoteStatus(username, statusIRI, true) + if err != nil { + return fmt.Errorf("DereferenceThread: error getting status with id %s: %s", statusIRI.String(), err) + } + + // first iterate up through ancestors, dereferencing if necessary as we go + if err := d.iterateAncestors(username, *statusIRI); err != nil { + return fmt.Errorf("error iterating ancestors of status %s: %s", statusIRI.String(), err) + } + + // now iterate down through descendants, again dereferencing as we go + if err := d.iterateDescendants(username, *statusIRI, statusable); err != nil { + return fmt.Errorf("error iterating descendants of status %s: %s", statusIRI.String(), err) + } + + return nil +} + +// iterateAncestors has the goal of reaching the oldest ancestor of a given status, and stashing all statuses along the way. +func (d *deref) iterateAncestors(username string, statusIRI url.URL) error { + l := d.log.WithFields(logrus.Fields{ + "func": "iterateAncestors", + "username": username, + "statusIRI": statusIRI.String(), + }) + l.Debug("entering iterateAncestors") + + // if it's our status we don't need to dereference anything so we can immediately move up the chain + if statusIRI.Host == d.config.Host { + l.Debug("iri belongs to us, moving up to next ancestor") + + // since this is our status, we know we can extract the id from the status path + _, id, err := util.ParseStatusesPath(&statusIRI) + if err != nil { + return err + } + + status := >smodel.Status{} + if err := d.db.GetByID(id, status); err != nil { + return err + } + + if status.InReplyToURI == "" { + // status doesn't reply to anything + return nil + } + nextIRI, err := url.Parse(status.URI) + if err != nil { + return err + } + return d.iterateAncestors(username, *nextIRI) + } + + // If we reach here, we're looking at a remote status -- make sure we have it in our db by calling GetRemoteStatus + // We call it with refresh to true because we want the statusable representation to parse inReplyTo from. + status, statusable, _, err := d.GetRemoteStatus(username, &statusIRI, true) + if err != nil { + l.Debugf("error getting remote status: %s", err) + return nil + } + + inReplyTo := ap.ExtractInReplyToURI(statusable) + if inReplyTo == nil || inReplyTo.String() == "" { + // status doesn't reply to anything + return nil + } + + // get the ancestor status into our database if we don't have it yet + if _, _, _, err := d.GetRemoteStatus(username, inReplyTo, false); err != nil { + l.Debugf("error getting remote status: %s", err) + return nil + } + + // now enrich the current status, since we should have the ancestor in the db + if _, err := d.EnrichRemoteStatus(username, status); err != nil { + l.Debugf("error enriching remote status: %s", err) + return nil + } + + // now move up to the next ancestor + return d.iterateAncestors(username, *inReplyTo) +} + +func (d *deref) iterateDescendants(username string, statusIRI url.URL, statusable ap.Statusable) error { + l := d.log.WithFields(logrus.Fields{ + "func": "iterateDescendants", + "username": username, + "statusIRI": statusIRI.String(), + }) + l.Debug("entering iterateDescendants") + + // if it's our status we already have descendants stashed so we can bail early + if statusIRI.Host == d.config.Host { + l.Debug("iri belongs to us, bailing") + return nil + } + + replies := statusable.GetActivityStreamsReplies() + if replies == nil || !replies.IsActivityStreamsCollection() { + l.Debug("no replies, bailing") + return nil + } + + repliesCollection := replies.GetActivityStreamsCollection() + if repliesCollection == nil { + l.Debug("replies collection is nil, bailing") + return nil + } + + first := repliesCollection.GetActivityStreamsFirst() + if first == nil { + l.Debug("replies collection has no first, bailing") + return nil + } + + firstPage := first.GetActivityStreamsCollectionPage() + if firstPage == nil { + l.Debug("first has no collection page, bailing") + return nil + } + + firstPageNext := firstPage.GetActivityStreamsNext() + if firstPageNext == nil || !firstPageNext.IsIRI() { + l.Debug("next is not an iri, bailing") + return nil + } + + var foundReplies int + currentPageIRI := firstPageNext.GetIRI() + +pageLoop: + for { + l.Debugf("dereferencing page %s", currentPageIRI) + nextPage, err := d.DereferenceCollectionPage(username, currentPageIRI) + if err != nil { + return nil + } + + // next items could be either a list of URLs or a list of statuses + + nextItems := nextPage.GetActivityStreamsItems() + if nextItems.Len() == 0 { + // no items on this page, which means we're done + break pageLoop + } + + // have a look through items and see what we can find + for iter := nextItems.Begin(); iter != nextItems.End(); iter = iter.Next() { + // We're looking for a url to feed to GetRemoteStatus. + // Items can be either an IRI, or a Note. + // If a note, we grab the ID from it and call it, rather than parsing the note. + + var itemURI *url.URL + if iter.IsIRI() { + // iri, easy + itemURI = iter.GetIRI() + } else if iter.IsActivityStreamsNote() { + // note, get the id from it to use as iri + n := iter.GetActivityStreamsNote() + id := n.GetJSONLDId() + if id != nil && id.IsIRI() { + itemURI = id.GetIRI() + } + } else { + // if it's not an iri or a note, we don't know how to process it + continue + } + + if itemURI.Host == d.config.Host { + // skip if the reply is from us -- we already have it then + continue + } + + // we can confidently say now that we found something + foundReplies = foundReplies + 1 + + // get the remote statusable and put it in the db + _, statusable, new, err := d.GetRemoteStatus(username, itemURI, false) + if new && err == nil && statusable != nil { + // now iterate descendants of *that* status + if err := d.iterateDescendants(username, *itemURI, statusable); err != nil { + continue + } + } + } + + next := nextPage.GetActivityStreamsNext() + if next != nil && next.IsIRI() { + l.Debug("setting next page") + currentPageIRI = next.GetIRI() + } else { + l.Debug("no next page, bailing") + break pageLoop + } + } + + l.Debugf("foundReplies %d", foundReplies) + return nil +} diff --git a/internal/federation/federatingdb/update.go b/internal/federation/federatingdb/update.go index e4a4920c8..3f4e3e413 100644 --- a/internal/federation/federatingdb/update.go +++ b/internal/federation/federatingdb/update.go @@ -9,8 +9,8 @@ import ( "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -78,7 +78,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error { typeName == gtsmodel.ActivityStreamsPerson || typeName == gtsmodel.ActivityStreamsService { // it's an UPDATE to some kind of account - var accountable typeutils.Accountable + var accountable ap.Accountable switch asType.GetTypeName() { case gtsmodel.ActivityStreamsApplication: diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go index 1acdb6cb1..9e21b43bf 100644 --- a/internal/federation/federatingprotocol.go +++ b/internal/federation/federatingprotocol.go @@ -31,7 +31,6 @@ import ( "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -139,7 +138,7 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } // we don't have an entry for this instance yet so dereference it - i, err = f.DereferenceRemoteInstance(username, &url.URL{ + i, err = f.GetRemoteInstance(username, &url.URL{ Scheme: publicKeyOwnerURI.Scheme, Host: publicKeyOwnerURI.Host, }) @@ -153,51 +152,9 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } } - requestingAccount := >smodel.Account{} - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: publicKeyOwnerURI.String()}}, requestingAccount); err != nil { - // there's been a proper error so return it - if _, ok := err.(db.ErrNoEntries); !ok { - return ctx, false, fmt.Errorf("error getting requesting account with public key id %s: %s", publicKeyOwnerURI.String(), err) - } - - // we don't know this account (yet) so let's dereference it right now - person, err := f.DereferenceRemoteAccount(requestedAccount.Username, publicKeyOwnerURI) - if err != nil { - return ctx, false, fmt.Errorf("error dereferencing account with public key id %s: %s", publicKeyOwnerURI.String(), err) - } - - a, err := f.typeConverter.ASRepresentationToAccount(person, false) - if err != nil { - return ctx, false, fmt.Errorf("error converting person with public key id %s to account: %s", publicKeyOwnerURI.String(), err) - } - - aID, err := id.NewRandomULID() - if err != nil { - return ctx, false, err - } - a.ID = aID - - if err := f.db.Put(a); err != nil { - l.Errorf("error inserting dereferenced remote account: %s", err) - } - - requestingAccount = a - - // send the newly dereferenced account into the processor channel for further async processing - fromFederatorChanI := ctx.Value(util.APFromFederatorChanKey) - if fromFederatorChanI == nil { - l.Error("from federator channel wasn't set on context") - } - fromFederatorChan, ok := fromFederatorChanI.(chan gtsmodel.FromFederator) - if !ok { - l.Error("from federator channel was set on context but couldn't be parsed") - } - - fromFederatorChan <- gtsmodel.FromFederator{ - APObjectType: gtsmodel.ActivityStreamsProfile, - APActivityType: gtsmodel.ActivityStreamsCreate, - GTSModel: requestingAccount, - } + requestingAccount, _, err := f.GetRemoteAccount(username, publicKeyOwnerURI, false) + if err != nil { + return nil, false, fmt.Errorf("couldn't get remote account: %s", err) } withRequester := context.WithValue(ctx, util.APRequestingAccount, requestingAccount) diff --git a/internal/federation/federator.go b/internal/federation/federator.go index a5ffb3de8..ea9e61831 100644 --- a/internal/federation/federator.go +++ b/internal/federation/federator.go @@ -21,12 +21,13 @@ package federation import ( "context" "net/url" - "sync" "github.com/go-fed/activity/pub" "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" @@ -40,6 +41,7 @@ type Federator interface { FederatingActor() pub.FederatingActor // FederatingDB returns the underlying FederatingDB interface. FederatingDB() federatingdb.DB + // AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources. // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. // @@ -49,29 +51,21 @@ type Federator interface { // // If something goes wrong during authentication, nil, false, and an error will be returned. AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, bool, error) + // FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that // account, or an error if it doesn't exist or can't be retrieved. FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) - // DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI). - // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. - DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) - // DereferenceRemoteStatus can be used to get the representation of a remote status, based on its ID (which is a URI). - // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. - DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) - // DereferenceRemoteInstance takes the URL of a remote instance, and a username (optional) to spin up a transport with. It then - // does its damnedest to get some kind of information back about the instance, trying /api/v1/instance, then /.well-known/nodeinfo - DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) - // DereferenceStatusFields does further dereferencing on a status. - DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error - // DereferenceAccountFields does further dereferencing on an account. - DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error - // DereferenceAnnounce does further dereferencing on an announce. + + DereferenceRemoteThread(username string, statusURI *url.URL) error DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error - // GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username. - // This can be used for making signed http requests. - // - // If username is an empty string, our instance user's credentials will be used instead. - GetTransportForUser(username string) (transport.Transport, error) + + GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) + + GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) + EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) + + GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) + // Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID. Handshaking(username string, remoteAccountID *url.URL) bool pub.CommonBehavior @@ -85,16 +79,17 @@ type federator struct { clock pub.Clock typeConverter typeutils.TypeConverter transportController transport.Controller + dereferencer dereferencing.Dereferencer mediaHandler media.Handler actor pub.FederatingActor log *logrus.Logger - handshakes map[string][]*url.URL - handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map } // NewFederator returns a new federator func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, config *config.Config, log *logrus.Logger, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator { + dereferencer := dereferencing.NewDereferencer(config, db, typeConverter, transportController, mediaHandler, log) + clock := &Clock{} f := &federator{ config: config, @@ -103,9 +98,9 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr clock: &Clock{}, typeConverter: typeConverter, transportController: transportController, + dereferencer: dereferencer, mediaHandler: mediaHandler, log: log, - handshakeSync: &sync.Mutex{}, } actor := newFederatingActor(f, f, federatingDB, clock) f.actor = actor diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go index 4ba0796cd..d74070487 100644 --- a/internal/federation/federator_test.go +++ b/internal/federation/federator_test.go @@ -69,7 +69,7 @@ func (suite *ProtocolTestSuite) SetupSuite() { } func (suite *ProtocolTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, suite.accounts) } @@ -87,7 +87,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() { // setup transport controller with a no-op client so we don't make external calls tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { return nil, nil - })) + }), suite.db) // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) @@ -152,7 +152,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() { StatusCode: 200, Body: r, }, nil - })) + }), suite.db) // now setup module being tested, with the mock transport controller federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage)) diff --git a/internal/federation/finger.go b/internal/federation/finger.go index 6c6e9f6dc..0ffc60e5a 100644 --- a/internal/federation/finger.go +++ b/internal/federation/finger.go @@ -34,7 +34,7 @@ func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsernam return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain) } - t, err := f.GetTransportForUser(requestingUsername) + t, err := f.transportController.NewTransportForUsername(requestingUsername) if err != nil { return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) } diff --git a/internal/federation/handshake.go b/internal/federation/handshake.go index 511e3e174..47c8a6c84 100644 --- a/internal/federation/handshake.go +++ b/internal/federation/handshake.go @@ -3,78 +3,5 @@ package federation import "net/url" func (f *federator) Handshaking(username string, remoteAccountID *url.URL) bool { - f.handshakeSync.Lock() - defer f.handshakeSync.Unlock() - - if f.handshakes == nil { - // handshakes isn't even initialized yet so we can't be handshaking with anyone - return false - } - - remoteIDs, ok := f.handshakes[username] - if !ok { - // user isn't handshaking with anyone, bail - return false - } - - for _, id := range remoteIDs { - if id.String() == remoteAccountID.String() { - // we are currently handshaking with the remote account, yep - return true - } - } - - // didn't find it which means we're not handshaking - return false -} - -func (f *federator) startHandshake(username string, remoteAccountID *url.URL) { - f.handshakeSync.Lock() - defer f.handshakeSync.Unlock() - - // lazily initialize handshakes - if f.handshakes == nil { - f.handshakes = make(map[string][]*url.URL) - } - - remoteIDs, ok := f.handshakes[username] - if !ok { - // there was nothing in there yet, so just add this entry and return - f.handshakes[username] = []*url.URL{remoteAccountID} - return - } - - // add the remote ID to the slice - remoteIDs = append(remoteIDs, remoteAccountID) - f.handshakes[username] = remoteIDs -} - -func (f *federator) stopHandshake(username string, remoteAccountID *url.URL) { - f.handshakeSync.Lock() - defer f.handshakeSync.Unlock() - - if f.handshakes == nil { - return - } - - remoteIDs, ok := f.handshakes[username] - if !ok { - // there was nothing in there yet anyway so just bail - return - } - - newRemoteIDs := []*url.URL{} - for _, id := range remoteIDs { - if id.String() != remoteAccountID.String() { - newRemoteIDs = append(newRemoteIDs, id) - } - } - - if len(newRemoteIDs) == 0 { - // there are no handshakes so just remove this user entry from the map and save a few bytes - delete(f.handshakes, username) - } else { - // there are still other handshakes ongoing - f.handshakes[username] = newRemoteIDs - } + return f.dereferencer.Handshaking(username, remoteAccountID) } diff --git a/internal/federation/transport.go b/internal/federation/transport.go index a92f66d25..ed28749a1 100644 --- a/internal/federation/transport.go +++ b/internal/federation/transport.go @@ -6,8 +6,6 @@ import ( "net/url" "github.com/go-fed/activity/pub" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -35,7 +33,6 @@ import ( // returned Transport so that any private credentials are able to be // garbage collected. func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofedAgent string) (pub.Transport, error) { - var username string var err error @@ -53,32 +50,5 @@ func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofe return nil, fmt.Errorf("id %s was neither an inbox path nor an outbox path", actorBoxIRI.String()) } - account := >smodel.Account{} - if err := f.db.GetLocalAccountByUsername(username, account); err != nil { - return nil, fmt.Errorf("error getting account with username %s from the db: %s", username, err) - } - - return f.transportController.NewTransport(account.PublicKeyURI, account.PrivateKey) -} - -func (f *federator) GetTransportForUser(username string) (transport.Transport, error) { - // We need an account to use to create a transport for dereferecing something. - // If a username has been given, we can fetch the account with that username and use it. - // Otherwise, we can take the instance account and use those credentials to make the request. - ourAccount := >smodel.Account{} - var u string - if username == "" { - u = f.config.Host - } else { - u = username - } - if err := f.db.GetLocalAccountByUsername(u, ourAccount); err != nil { - return nil, fmt.Errorf("error getting account %s from db: %s", username, err) - } - - transport, err := f.transportController.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey) - if err != nil { - return nil, fmt.Errorf("error creating transport for user %s: %s", username, err) - } - return transport, nil + return f.transportController.NewTransportForUsername(username) } diff --git a/internal/gtsmodel/activitystreams.go b/internal/gtsmodel/activitystreams.go index 77c935c5f..5cd92015c 100644 --- a/internal/gtsmodel/activitystreams.go +++ b/internal/gtsmodel/activitystreams.go @@ -43,6 +43,10 @@ const ( ActivityStreamsTombstone = "Tombstone" // ActivityStreamsVideo https://www.w3.org/TR/activitystreams-vocabulary/#dfn-video ActivityStreamsVideo = "Video" + //ActivityStreamsCollection https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection + ActivityStreamsCollection = "Collection" + // ActivityStreamsCollectionPage https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collectionpage + ActivityStreamsCollectionPage = "CollectionPage" ) const ( diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index 524b9c3ef..106298bcd 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -18,7 +18,9 @@ package gtsmodel -import "time" +import ( + "time" +) // Status represents a user-created 'post' or 'status' in the database, either remote or local type Status struct { diff --git a/internal/processing/account/get.go b/internal/processing/account/get.go index b937ace5b..d2994e246 100644 --- a/internal/processing/account/get.go +++ b/internal/processing/account/get.go @@ -36,15 +36,6 @@ func (p *processor) Get(requestingAccount *gtsmodel.Account, targetAccountID str return nil, fmt.Errorf("db error: %s", err) } - // lazily dereference things on the account if it hasn't been done yet - var requestingUsername string - if requestingAccount != nil { - requestingUsername = requestingAccount.Username - } - if err := p.federator.DereferenceAccountFields(targetAccount, requestingUsername, false); err != nil { - p.log.WithField("func", "AccountGet").Debugf("dereferencing account: %s", err) - } - var blocked bool var err error if requestingAccount != nil { diff --git a/internal/processing/account/getfollowers.go b/internal/processing/account/getfollowers.go index bfc463d3f..0806a82c0 100644 --- a/internal/processing/account/getfollowers.go +++ b/internal/processing/account/getfollowers.go @@ -63,12 +63,6 @@ func (p *processor) FollowersGet(requestingAccount *gtsmodel.Account, targetAcco return nil, gtserror.NewErrorInternalError(err) } - // derefence account fields in case we haven't done it already - if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil { - // don't bail if we can't fetch them, we'll try another time - p.log.WithField("func", "AccountFollowersGet").Debugf("error dereferencing account fields: %s", err) - } - account, err := p.tc.AccountToMastoPublic(a) if err != nil { return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/account/getfollowing.go b/internal/processing/account/getfollowing.go index bb6a905f4..75e89dacb 100644 --- a/internal/processing/account/getfollowing.go +++ b/internal/processing/account/getfollowing.go @@ -63,12 +63,6 @@ func (p *processor) FollowingGet(requestingAccount *gtsmodel.Account, targetAcco return nil, gtserror.NewErrorInternalError(err) } - // derefence account fields in case we haven't done it already - if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil { - // don't bail if we can't fetch them, we'll try another time - p.log.WithField("func", "AccountFollowingGet").Debugf("error dereferencing account fields: %s", err) - } - account, err := p.tc.AccountToMastoPublic(a) if err != nil { return nil, gtserror.NewErrorInternalError(err) diff --git a/internal/processing/federation.go b/internal/processing/federation.go index 966dab08d..765fdf862 100644 --- a/internal/processing/federation.go +++ b/internal/processing/federation.go @@ -31,65 +31,9 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/util" ) -// dereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given -// username to perform the validation. It will *also* dereference the originator of the request and return it as a gtsmodel account -// for further processing. NOTE that this function will have the side effect of putting the dereferenced account into the database, -// and passing it into the processor through a channel for further asynchronous processing. -func (p *processor) dereferenceFediRequest(username string, requestingAccountURI *url.URL) (*gtsmodel.Account, error) { - // OK now we can do the dereferencing part - // we might already have an entry for this account so check that first - requestingAccount := >smodel.Account{} - - err := p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount) - if err == nil { - // we do have it yay, return it - return requestingAccount, nil - } - - if _, ok := err.(db.ErrNoEntries); !ok { - // something has actually gone wrong so bail - return nil, fmt.Errorf("database error getting account with uri %s: %s", requestingAccountURI.String(), err) - } - - // we just don't have an entry for this account yet - // what we do now should depend on our chosen federation method - // for now though, we'll just dereference it - // TODO: slow-fed - requestingPerson, err := p.federator.DereferenceRemoteAccount(username, requestingAccountURI) - if err != nil { - return nil, fmt.Errorf("couldn't dereference %s: %s", requestingAccountURI.String(), err) - } - - // convert it to our internal account representation - requestingAccount, err = p.tc.ASRepresentationToAccount(requestingPerson, false) - if err != nil { - return nil, fmt.Errorf("couldn't convert dereferenced uri %s to gtsmodel account: %s", requestingAccountURI.String(), err) - } - - requestingAccountID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - requestingAccount.ID = requestingAccountID - - if err := p.db.Put(requestingAccount); err != nil { - return nil, fmt.Errorf("database error inserting account with uri %s: %s", requestingAccountURI.String(), err) - } - - // put it in our channel to queue it for async processing - p.fromFederator <- gtsmodel.FromFederator{ - APObjectType: gtsmodel.ActivityStreamsProfile, - APActivityType: gtsmodel.ActivityStreamsCreate, - GTSModel: requestingAccount, - } - - return requestingAccount, nil -} - func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, requestURL *url.URL) (interface{}, gtserror.WithCode) { // get the account the request is referring to requestedAccount := >smodel.Account{} @@ -112,9 +56,9 @@ func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, r return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - // if we're already handshaking/dereferencing a remote account, we can skip the dereferencing part + // if we're not already handshaking/dereferencing a remote account, dereference it now if !p.federator.Handshaking(requestedUsername, requestingAccountURI) { - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -158,7 +102,7 @@ func (p *processor) GetFediFollowers(ctx context.Context, requestedUsername stri return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -203,7 +147,7 @@ func (p *processor) GetFediFollowing(ctx context.Context, requestedUsername stri return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -248,7 +192,7 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string, return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") } - requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI) + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) if err != nil { return nil, gtserror.NewErrorNotAuthorized(err) } @@ -295,6 +239,139 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string, return data, nil } +func (p *processor) GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode) { + // get the account the request is referring to + requestedAccount := >smodel.Account{} + if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err)) + } + + // authenticate the request + requestingAccountURI, authenticated, err := p.federator.AuthenticateFederatedRequest(ctx, requestedUsername) + if err != nil || !authenticated { + return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized") + } + + requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false) + if err != nil { + return nil, gtserror.NewErrorNotAuthorized(err) + } + + // authorize the request: + // 1. check if a block exists between the requester and the requestee + blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + + if blocked { + return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID)) + } + + // get the status out of the database here + s := >smodel.Status{} + if err := p.db.GetWhere([]db.Where{ + {Key: "id", Value: requestedStatusID}, + {Key: "account_id", Value: requestedAccount.ID}, + }, s); err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting status with id %s and account id %s: %s", requestedStatusID, requestedAccount.ID, err)) + } + + visible, err := p.filter.StatusVisible(s, requestingAccount) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + if !visible { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("status with id %s not visible to user with id %s", s.ID, requestingAccount.ID)) + } + + var data map[string]interface{} + + // now there are three scenarios: + // 1. we're asked for the whole collection and not a page -- we can just return the collection, with no items, but a link to 'first' page. + // 2. we're asked for a page but only_other_accounts has not been set in the query -- so we should just return the first page of the collection, with no items. + // 3. we're asked for a page, and only_other_accounts has been set, and min_id has optionally been set -- so we need to return some actual items! + + if !page { + // scenario 1 + + // get the collection + collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + + data, err = streams.Serialize(collection) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } else if page && requestURL.Query().Get("only_other_accounts") == "" { + // scenario 2 + + // get the collection + collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + // but only return the first page + data, err = streams.Serialize(collection.GetActivityStreamsFirst().GetActivityStreamsCollectionPage()) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } else { + // scenario 3 + // get immediate children + replies, err := p.db.StatusChildren(s, true, minID) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + + // filter children and extract URIs + replyURIs := map[string]*url.URL{} + for _, r := range replies { + // only show public or unlocked statuses as replies + if r.Visibility != gtsmodel.VisibilityPublic && r.Visibility != gtsmodel.VisibilityUnlocked { + continue + } + + // respect onlyOtherAccounts parameter + if onlyOtherAccounts && r.AccountID == requestedAccount.ID { + continue + } + + // only show replies that the status owner can see + visibleToStatusOwner, err := p.filter.StatusVisible(r, requestedAccount) + if err != nil || !visibleToStatusOwner { + continue + } + + // only show replies that the requester can see + visibleToRequester, err := p.filter.StatusVisible(r, requestingAccount) + if err != nil || !visibleToRequester { + continue + } + + rURI, err := url.Parse(r.URI) + if err != nil { + continue + } + + replyURIs[r.ID] = rURI + } + + repliesPage, err := p.tc.StatusURIsToASRepliesPage(s, onlyOtherAccounts, minID, replyURIs) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + data, err = streams.Serialize(repliesPage) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } + + return data, nil +} + func (p *processor) GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode) { // get the account the request is referring to requestedAccount := >smodel.Account{} diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index 94a4e5af8..949a734c7 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -21,6 +21,7 @@ package processing import ( "errors" "fmt" + "net/url" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -47,36 +48,21 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er return errors.New("note was not parseable as *gtsmodel.Status") } - l.Trace("will now derefence incoming status") - if err := p.federator.DereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil { - return fmt.Errorf("error dereferencing status from federator: %s", err) - } - if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil { - return fmt.Errorf("error updating dereferenced status in the db: %s", err) - } - - if err := p.timelineStatus(incomingStatus); err != nil { + status, err := p.federator.EnrichRemoteStatus(federatorMsg.ReceivingAccount.Username, incomingStatus) + if err != nil { return err } - if err := p.notifyStatus(incomingStatus); err != nil { + if err := p.timelineStatus(status); err != nil { return err } + if err := p.notifyStatus(status); err != nil { + return err + } case gtsmodel.ActivityStreamsProfile: // CREATE AN ACCOUNT - incomingAccount, ok := federatorMsg.GTSModel.(*gtsmodel.Account) - if !ok { - return errors.New("profile was not parseable as *gtsmodel.Account") - } - - l.Trace("will now derefence incoming account") - if err := p.federator.DereferenceAccountFields(incomingAccount, "", false); err != nil { - return fmt.Errorf("error dereferencing account from federator: %s", err) - } - if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil { - return fmt.Errorf("error updating dereferenced account in the db: %s", err) - } + // nothing to do here case gtsmodel.ActivityStreamsLike: // CREATE A FAVE incomingFave, ok := federatorMsg.GTSModel.(*gtsmodel.StatusFave) @@ -154,12 +140,13 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er return errors.New("profile was not parseable as *gtsmodel.Account") } - l.Trace("will now derefence incoming account") - if err := p.federator.DereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil { - return fmt.Errorf("error dereferencing account from federator: %s", err) + incomingAccountURI, err := url.Parse(incomingAccount.URI) + if err != nil { + return err } - if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil { - return fmt.Errorf("error updating dereferenced account in the db: %s", err) + + if _, _, err := p.federator.GetRemoteAccount(federatorMsg.ReceivingAccount.Username, incomingAccountURI, true); err != nil { + return fmt.Errorf("error dereferencing account from federator: %s", err) } } case gtsmodel.ActivityStreamsDelete: diff --git a/internal/processing/processor.go b/internal/processing/processor.go index a09a370e9..16f9ac2a3 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -191,6 +191,10 @@ type Processor interface { // authentication before returning a JSON serializable interface to the caller. GetFediStatus(ctx context.Context, requestedUsername string, requestedStatusID string, requestURL *url.URL) (interface{}, gtserror.WithCode) + // GetFediStatus handles the getting of a fedi/activitypub representation of replies to a status, performing appropriate + // authentication before returning a JSON serializable interface to the caller. + GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode) + // GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups. GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode) diff --git a/internal/processing/search.go b/internal/processing/search.go index 727ad13bd..737ad8f71 100644 --- a/internal/processing/search.go +++ b/internal/processing/search.go @@ -19,7 +19,6 @@ package processing import ( - "errors" "fmt" "net/url" "strings" @@ -29,7 +28,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -122,6 +120,11 @@ func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQu } func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) { + l := p.log.WithFields(logrus.Fields{ + "func": "searchStatusByURI", + "uri": uri.String(), + "resolve": resolve, + }) maybeStatus := >smodel.Status{} if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil { @@ -134,57 +137,12 @@ func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve // we don't have it locally so dereference it if we're allowed to if resolve { - statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri) + status, _, _, err := p.federator.GetRemoteStatus(authed.Account.Username, uri, true) if err == nil { - // it IS a status! - - // extract the status owner's IRI from the statusable - var statusOwnerURI *url.URL - statusAttributedTo := statusable.GetActivityStreamsAttributedTo() - for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() { - if i.IsIRI() { - statusOwnerURI = i.GetIRI() - break - } + if err := p.federator.DereferenceRemoteThread(authed.Account.Username, uri); err != nil { + // try to deref the thread while we're here + l.Debugf("searchStatusByURI: error dereferencing remote thread: %s", err) } - if statusOwnerURI == nil { - return nil, errors.New("couldn't extract ownerAccountURI from statusable") - } - - // make sure the status owner exists in the db by searching for it - _, err := p.searchAccountByURI(authed, statusOwnerURI, resolve) - if err != nil { - return nil, err - } - - // we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly - - // first turn it into a gtsmodel.Status - status, err := p.tc.ASStatusToStatus(statusable) - if err != nil { - return nil, gtserror.NewErrorInternalError(err) - } - - statusID, err := id.NewULIDFromTime(status.CreatedAt) - if err != nil { - return nil, err - } - status.ID = statusID - - if err := p.db.Put(status); err != nil { - return nil, fmt.Errorf("error putting status in the db: %s", err) - } - - // properly dereference everything in the status (media attachments etc) - if err := p.federator.DereferenceStatusFields(status, authed.Account.Username); err != nil { - return nil, fmt.Errorf("error dereferencing status fields: %s", err) - } - - // update with the nicely dereferenced status - if err := p.db.UpdateByID(status.ID, status); err != nil { - return nil, fmt.Errorf("error updating status in the db: %s", err) - } - return status, nil } } @@ -202,31 +160,10 @@ func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve } if resolve { // we don't have it locally so try and dereference it - accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri) + account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, uri, true) if err != nil { return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err) } - - // it IS an account! - account, err := p.tc.ASRepresentationToAccount(accountable, false) - if err != nil { - return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err) - } - - accountID, err := id.NewRandomULID() - if err != nil { - return nil, err - } - account.ID = accountID - - if err := p.db.Put(account); err != nil { - return nil, fmt.Errorf("searchAccountByURI: error inserting account with uri %s: %s", uri.String(), err) - } - - if err := p.federator.DereferenceAccountFields(account, authed.Account.Username, false); err != nil { - return nil, fmt.Errorf("searchAccountByURI: error further dereferencing account with uri %s: %s", uri.String(), err) - } - return account, nil } return nil, nil @@ -275,35 +212,12 @@ func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, r return nil, fmt.Errorf("searchAccountByMention: error fingering remote account with username %s and domain %s: %s", username, domain, err) } - // dereference the account based on the URI we retrieved from the webfinger lookup - accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI) + // we don't have it locally so try and dereference it + account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, acctURI, true) if err != nil { - // something went wrong doing the dereferencing so we can't process the request - return nil, fmt.Errorf("searchAccountByMention: error dereferencing remote account with uri %s: %s", acctURI.String(), err) - } - - // convert the dereferenced account to the gts model of that account - foundAccount, err := p.tc.ASRepresentationToAccount(accountable, false) - if err != nil { - // something went wrong doing the conversion to a gtsmodel.Account so we can't process the request - return nil, fmt.Errorf("searchAccountByMention: error converting account with uri %s: %s", acctURI.String(), err) - } - - foundAccountID, err := id.NewULID() - if err != nil { - return nil, err - } - foundAccount.ID = foundAccountID - - // put this new account in our database - if err := p.db.Put(foundAccount); err != nil { - return nil, fmt.Errorf("searchAccountByMention: error inserting account with uri %s: %s", acctURI.String(), err) - } - - // properly dereference all the fields on the account immediately - if err := p.federator.DereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil { - return nil, fmt.Errorf("searchAccountByMention: error dereferencing fields on account with uri %s: %s", acctURI.String(), err) + return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err) } + return account, nil } return nil, nil diff --git a/internal/processing/status/context.go b/internal/processing/status/context.go index 72b9b5623..32c528296 100644 --- a/internal/processing/status/context.go +++ b/internal/processing/status/context.go @@ -33,7 +33,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (* return nil, gtserror.NewErrorForbidden(fmt.Errorf("account with id %s does not have permission to view status %s", account.ID, targetStatusID)) } - parents, err := p.db.StatusParents(targetStatus) + parents, err := p.db.StatusParents(targetStatus, false) if err != nil { return nil, gtserror.NewErrorInternalError(err) } @@ -51,7 +51,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (* return context.Ancestors[i].ID < context.Ancestors[j].ID }) - children, err := p.db.StatusChildren(targetStatus) + children, err := p.db.StatusChildren(targetStatus, false, "") if err != nil { return nil, gtserror.NewErrorInternalError(err) } diff --git a/internal/text/link_test.go b/internal/text/link_test.go index 15e27f870..83c42f045 100644 --- a/internal/text/link_test.go +++ b/internal/text/link_test.go @@ -86,7 +86,7 @@ func (suite *LinkTestSuite) SetupTest() { suite.log = testrig.NewTestLog() suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } func (suite *LinkTestSuite) TearDownTest() { diff --git a/internal/text/plain_test.go b/internal/text/plain_test.go index 1e0d1471a..183ccc478 100644 --- a/internal/text/plain_test.go +++ b/internal/text/plain_test.go @@ -57,7 +57,7 @@ func (suite *PlainTestSuite) SetupTest() { suite.log = testrig.NewTestLog() suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log) - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } func (suite *PlainTestSuite) TearDownTest() { diff --git a/internal/transport/controller.go b/internal/transport/controller.go index c01af0900..07d20cdcf 100644 --- a/internal/transport/controller.go +++ b/internal/transport/controller.go @@ -27,15 +27,19 @@ import ( "github.com/go-fed/httpsig" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) // Controller generates transports for use in making federation requests to other servers. type Controller interface { NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error) + NewTransportForUsername(username string) (Transport, error) } type controller struct { config *config.Config + db db.DB clock pub.Clock client pub.HttpClient appAgent string @@ -43,9 +47,10 @@ type controller struct { } // NewController returns an implementation of the Controller interface for creating new transports -func NewController(config *config.Config, clock pub.Clock, client pub.HttpClient, log *logrus.Logger) Controller { +func NewController(config *config.Config, db db.DB, clock pub.Clock, client pub.HttpClient, log *logrus.Logger) Controller { return &controller{ config: config, + db: db, clock: clock, client: client, appAgent: fmt.Sprintf("%s %s", config.ApplicationName, config.Host), @@ -55,10 +60,10 @@ func NewController(config *config.Config, clock pub.Clock, client pub.HttpClient // NewTransport returns a new http signature transport with the given public key id (a URL), and the given private key. func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error) { - prefs := []httpsig.Algorithm{httpsig.RSA_SHA256, httpsig.RSA_SHA512} + prefs := []httpsig.Algorithm{httpsig.RSA_SHA512} digestAlgo := httpsig.DigestSha256 - getHeaders := []string{"(request-target)", "host", "date"} - postHeaders := []string{"(request-target)", "host", "date", "digest"} + getHeaders := []string{httpsig.RequestTarget, "host", "date"} + postHeaders := []string{httpsig.RequestTarget, "host", "date", "digest"} getSigner, _, err := httpsig.NewSigner(prefs, digestAlgo, getHeaders, httpsig.Signature, 120) if err != nil { @@ -85,3 +90,25 @@ func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (T log: c.log, }, nil } + +func (c *controller) NewTransportForUsername(username string) (Transport, error) { + // We need an account to use to create a transport for dereferecing something. + // If a username has been given, we can fetch the account with that username and use it. + // Otherwise, we can take the instance account and use those credentials to make the request. + ourAccount := >smodel.Account{} + var u string + if username == "" { + u = c.config.Host + } else { + u = username + } + if err := c.db.GetLocalAccountByUsername(u, ourAccount); err != nil { + return nil, fmt.Errorf("error getting account %s from db: %s", username, err) + } + + transport, err := c.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey) + if err != nil { + return nil, fmt.Errorf("error creating transport for user %s: %s", username, err) + } + return transport, nil +} diff --git a/internal/typeutils/asinterfaces.go b/internal/typeutils/asinterfaces.go deleted file mode 100644 index d0b1cf617..000000000 --- a/internal/typeutils/asinterfaces.go +++ /dev/null @@ -1,265 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package typeutils - -import "github.com/go-fed/activity/streams/vocab" - -// Accountable represents the minimum activitypub interface for representing an 'account'. -// This interface is fulfilled by: Person, Application, Organization, Service, and Group -type Accountable interface { - withJSONLDId - withTypeName - - withPreferredUsername - withIcon - withName - withImage - withSummary - withDiscoverable - withURL - withPublicKey - withInbox - withOutbox - withFollowing - withFollowers - withFeatured -} - -// Statusable represents the minimum activitypub interface for representing a 'status'. -// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile -type Statusable interface { - withJSONLDId - withTypeName - - withSummary - withInReplyTo - withPublished - withURL - withAttributedTo - withTo - withCC - withSensitive - withConversation - withContent - withAttachment - withTag - withReplies -} - -// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'. -// This interface is fulfilled by: Audio, Document, Image, Video -type Attachmentable interface { - withTypeName - withMediaType - withURL - withName -} - -// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag. -type Hashtaggable interface { - withTypeName - withHref - withName -} - -// Emojiable represents the minimum interface for an 'emoji' tag. -type Emojiable interface { - withJSONLDId - withTypeName - withName - withUpdated - withIcon -} - -// Mentionable represents the minimum interface for a 'mention' tag. -type Mentionable interface { - withName - withHref -} - -// Followable represents the minimum interface for an activitystreams 'follow' activity. -type Followable interface { - withJSONLDId - withTypeName - - withActor - withObject -} - -// Likeable represents the minimum interface for an activitystreams 'like' activity. -type Likeable interface { - withJSONLDId - withTypeName - - withActor - withObject -} - -// Blockable represents the minimum interface for an activitystreams 'block' activity. -type Blockable interface { - withJSONLDId - withTypeName - - withActor - withObject -} - -// Announceable represents the minimum interface for an activitystreams 'announce' activity. -type Announceable interface { - withJSONLDId - withTypeName - - withActor - withObject - withPublished - withTo - withCC -} - -type withJSONLDId interface { - GetJSONLDId() vocab.JSONLDIdProperty -} - -type withTypeName interface { - GetTypeName() string -} - -type withPreferredUsername interface { - GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty -} - -type withIcon interface { - GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty -} - -type withName interface { - GetActivityStreamsName() vocab.ActivityStreamsNameProperty -} - -type withImage interface { - GetActivityStreamsImage() vocab.ActivityStreamsImageProperty -} - -type withSummary interface { - GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty -} - -type withDiscoverable interface { - GetTootDiscoverable() vocab.TootDiscoverableProperty -} - -type withURL interface { - GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty -} - -type withPublicKey interface { - GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty -} - -type withInbox interface { - GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty -} - -type withOutbox interface { - GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty -} - -type withFollowing interface { - GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty -} - -type withFollowers interface { - GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty -} - -type withFeatured interface { - GetTootFeatured() vocab.TootFeaturedProperty -} - -type withAttributedTo interface { - GetActivityStreamsAttributedTo() vocab.ActivityStreamsAttributedToProperty -} - -type withAttachment interface { - GetActivityStreamsAttachment() vocab.ActivityStreamsAttachmentProperty -} - -type withTo interface { - GetActivityStreamsTo() vocab.ActivityStreamsToProperty -} - -type withInReplyTo interface { - GetActivityStreamsInReplyTo() vocab.ActivityStreamsInReplyToProperty -} - -type withCC interface { - GetActivityStreamsCc() vocab.ActivityStreamsCcProperty -} - -type withSensitive interface { - // TODO -} - -type withConversation interface { - // TODO -} - -type withContent interface { - GetActivityStreamsContent() vocab.ActivityStreamsContentProperty -} - -type withPublished interface { - GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty -} - -type withTag interface { - GetActivityStreamsTag() vocab.ActivityStreamsTagProperty -} - -type withReplies interface { - GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty -} - -type withMediaType interface { - GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty -} - -// type withBlurhash interface { -// GetTootBlurhashProperty() vocab.TootBlurhashProperty -// } - -// type withFocalPoint interface { -// // TODO -// } - -type withHref interface { - GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty -} - -type withUpdated interface { - GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty -} - -type withActor interface { - GetActivityStreamsActor() vocab.ActivityStreamsActorProperty -} - -type withObject interface { - GetActivityStreamsObject() vocab.ActivityStreamsObjectProperty -} diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index 394de6e82..f754d282a 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -24,11 +24,12 @@ import ( "net/url" "strings" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (c *converter) ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error) { +func (c *converter) ASRepresentationToAccount(accountable ap.Accountable, update bool) (*gtsmodel.Account, error) { // first check if we actually already know this account uriProp := accountable.GetJSONLDId() if uriProp == nil || !uriProp.IsIRI() { @@ -55,7 +56,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // Username aka preferredUsername // We need this one so bail if it's not set. - username, err := extractPreferredUsername(accountable) + username, err := ap.ExtractPreferredUsername(accountable) if err != nil { return nil, fmt.Errorf("couldn't extract username: %s", err) } @@ -66,27 +67,27 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // avatar aka icon // if this one isn't extractable in a format we recognise we'll just skip it - if avatarURL, err := extractIconURL(accountable); err == nil { + if avatarURL, err := ap.ExtractIconURL(accountable); err == nil { acct.AvatarRemoteURL = avatarURL.String() } // header aka image // if this one isn't extractable in a format we recognise we'll just skip it - if headerURL, err := extractImageURL(accountable); err == nil { + if headerURL, err := ap.ExtractImageURL(accountable); err == nil { acct.HeaderRemoteURL = headerURL.String() } // display name aka name // we default to the username, but take the more nuanced name property if it exists acct.DisplayName = username - if displayName, err := extractName(accountable); err == nil { + if displayName, err := ap.ExtractName(accountable); err == nil { acct.DisplayName = displayName } // TODO: fields aka attachment array // note aka summary - note, err := extractSummary(accountable) + note, err := ap.ExtractSummary(accountable) if err == nil && note != "" { acct.Note = note } @@ -110,13 +111,13 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // discoverable // default to false -- take custom value if it's set though acct.Discoverable = false - discoverable, err := extractDiscoverable(accountable) + discoverable, err := ap.ExtractDiscoverable(accountable) if err == nil { acct.Discoverable = discoverable } // url property - url, err := extractURL(accountable) + url, err := ap.ExtractURL(accountable) if err == nil { // take the URL if we can find it acct.URL = url.String() @@ -155,7 +156,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo // TODO: alsoKnownAs // publicKey - pkey, pkeyURL, err := extractPublicKeyForOwner(accountable, uri) + pkey, pkeyURL, err := ap.ExtractPublicKeyForOwner(accountable, uri) if err != nil { return nil, fmt.Errorf("couldn't get public key for person %s: %s", uri.String(), err) } @@ -165,7 +166,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo return acct, nil } -func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error) { +func (c *converter) ASStatusToStatus(statusable ap.Statusable) (*gtsmodel.Status, error) { status := >smodel.Status{} // uri at which this status is reachable @@ -176,49 +177,49 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e status.URI = uriProp.GetIRI().String() // web url for viewing this status - if statusURL, err := extractURL(statusable); err == nil { + if statusURL, err := ap.ExtractURL(statusable); err == nil { status.URL = statusURL.String() } // the html-formatted content of this status - if content, err := extractContent(statusable); err == nil { + if content, err := ap.ExtractContent(statusable); err == nil { status.Content = content } // attachments to dereference and fetch later on (we don't do that here) - if attachments, err := extractAttachments(statusable); err == nil { + if attachments, err := ap.ExtractAttachments(statusable); err == nil { status.GTSMediaAttachments = attachments } // hashtags to dereference later on - if hashtags, err := extractHashtags(statusable); err == nil { + if hashtags, err := ap.ExtractHashtags(statusable); err == nil { status.GTSTags = hashtags } // emojis to dereference and fetch later on - if emojis, err := extractEmojis(statusable); err == nil { + if emojis, err := ap.ExtractEmojis(statusable); err == nil { status.GTSEmojis = emojis } // mentions to dereference later on - if mentions, err := extractMentions(statusable); err == nil { + if mentions, err := ap.ExtractMentions(statusable); err == nil { status.GTSMentions = mentions } // cw string for this status - if cw, err := extractSummary(statusable); err == nil { + if cw, err := ap.ExtractSummary(statusable); err == nil { status.ContentWarning = cw } // when was this status created? - published, err := extractPublished(statusable) + published, err := ap.ExtractPublished(statusable) if err == nil { status.CreatedAt = published } // which account posted this status? // if we don't know the account yet we can dereference it later - attributedTo, err := extractAttributedTo(statusable) + attributedTo, err := ap.ExtractAttributedTo(statusable) if err != nil { return nil, errors.New("attributedTo was empty") } @@ -233,8 +234,8 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e status.GTSAuthorAccount = statusOwner // check if there's a post that this is a reply to - inReplyToURI, err := extractInReplyToURI(statusable) - if err == nil { + inReplyToURI := ap.ExtractInReplyToURI(statusable) + if inReplyToURI != nil { // something is set so we can at least set this field on the // status and dereference using this later if we need to status.InReplyToURI = inReplyToURI.String() @@ -259,12 +260,12 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e // visibility entry for this status var visibility gtsmodel.Visibility - to, err := extractTos(statusable) + to, err := ap.ExtractTos(statusable) if err != nil { return nil, fmt.Errorf("error extracting TO values: %s", err) } - cc, err := extractCCs(statusable) + cc, err := ap.ExtractCCs(statusable) if err != nil { return nil, fmt.Errorf("error extracting CC values: %s", err) } @@ -315,7 +316,7 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e return status, nil } -func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error) { +func (c *converter) ASFollowToFollowRequest(followable ap.Followable) (*gtsmodel.FollowRequest, error) { idProp := followable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { @@ -323,7 +324,7 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo } uri := idProp.GetIRI().String() - origin, err := extractActor(followable) + origin, err := ap.ExtractActor(followable) if err != nil { return nil, errors.New("error extracting actor property from follow") } @@ -332,7 +333,7 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(followable) + target, err := ap.ExtractObject(followable) if err != nil { return nil, errors.New("error extracting object property from follow") } @@ -350,14 +351,14 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo return followRequest, nil } -func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, error) { +func (c *converter) ASFollowToFollow(followable ap.Followable) (*gtsmodel.Follow, error) { idProp := followable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { return nil, errors.New("no id property set on follow, or was not an iri") } uri := idProp.GetIRI().String() - origin, err := extractActor(followable) + origin, err := ap.ExtractActor(followable) if err != nil { return nil, errors.New("error extracting actor property from follow") } @@ -366,7 +367,7 @@ func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, e return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(followable) + target, err := ap.ExtractObject(followable) if err != nil { return nil, errors.New("error extracting object property from follow") } @@ -384,14 +385,14 @@ func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, e return follow, nil } -func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error) { +func (c *converter) ASLikeToFave(likeable ap.Likeable) (*gtsmodel.StatusFave, error) { idProp := likeable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { return nil, errors.New("no id property set on like, or was not an iri") } uri := idProp.GetIRI().String() - origin, err := extractActor(likeable) + origin, err := ap.ExtractActor(likeable) if err != nil { return nil, errors.New("error extracting actor property from like") } @@ -400,7 +401,7 @@ func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(likeable) + target, err := ap.ExtractObject(likeable) if err != nil { return nil, errors.New("error extracting object property from like") } @@ -426,14 +427,14 @@ func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error }, nil } -func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) { +func (c *converter) ASBlockToBlock(blockable ap.Blockable) (*gtsmodel.Block, error) { idProp := blockable.GetJSONLDId() if idProp == nil || !idProp.IsIRI() { return nil, errors.New("ASBlockToBlock: no id property set on block, or was not an iri") } uri := idProp.GetIRI().String() - origin, err := extractActor(blockable) + origin, err := ap.ExtractActor(blockable) if err != nil { return nil, errors.New("ASBlockToBlock: error extracting actor property from block") } @@ -442,7 +443,7 @@ func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) return nil, fmt.Errorf("ASBlockToBlock: error extracting account with uri %s from the database: %s", origin.String(), err) } - target, err := extractObject(blockable) + target, err := ap.ExtractObject(blockable) if err != nil { return nil, errors.New("ASBlockToBlock: error extracting object property from block") } @@ -461,7 +462,7 @@ func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) }, nil } -func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Status, bool, error) { +func (c *converter) ASAnnounceToStatus(announceable ap.Announceable) (*gtsmodel.Status, bool, error) { status := >smodel.Status{} isNew := true @@ -480,7 +481,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta status.URI = uri // get the URI of the announced/boosted status - boostedStatusURI, err := extractObject(announceable) + boostedStatusURI, err := ap.ExtractObject(announceable) if err != nil { return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error getting object from announce: %s", err) } @@ -491,7 +492,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta } // get the published time for the announce - published, err := extractPublished(announceable) + published, err := ap.ExtractPublished(announceable) if err != nil { return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting published time: %s", err) } @@ -499,7 +500,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta status.UpdatedAt = published // get the actor's IRI (ie., the person who boosted the status) - actor, err := extractActor(announceable) + actor, err := ap.ExtractActor(announceable) if err != nil { return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting actor: %s", err) } @@ -522,12 +523,12 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta // parse the visibility from the To and CC entries var visibility gtsmodel.Visibility - to, err := extractTos(announceable) + to, err := ap.ExtractTos(announceable) if err != nil { return nil, isNew, fmt.Errorf("error extracting TO values: %s", err) } - cc, err := extractCCs(announceable) + cc, err := ap.ExtractCCs(announceable) if err != nil { return nil, isNew, fmt.Errorf("error extracting CC values: %s", err) } diff --git a/internal/typeutils/astointernal_test.go b/internal/typeutils/astointernal_test.go index 9d6ce4e0a..2e33271c5 100644 --- a/internal/typeutils/astointernal_test.go +++ b/internal/typeutils/astointernal_test.go @@ -28,6 +28,7 @@ import ( "github.com/go-fed/activity/streams/vocab" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -342,7 +343,7 @@ func (suite *ASToInternalTestSuite) SetupSuite() { } func (suite *ASToInternalTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } func (suite *ASToInternalTestSuite) TestParsePerson() { @@ -364,7 +365,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() { t, err := streams.ToType(context.Background(), m) assert.NoError(suite.T(), err) - rep, ok := t.(typeutils.Accountable) + rep, ok := t.(ap.Accountable) assert.True(suite.T(), ok) acct, err := suite.typeconverter.ASRepresentationToAccount(rep, false) @@ -391,7 +392,7 @@ func (suite *ASToInternalTestSuite) TestParseStatus() { first := obj.Begin() assert.NotNil(suite.T(), first) - rep, ok := first.GetType().(typeutils.Statusable) + rep, ok := first.GetType().(ap.Statusable) assert.True(suite.T(), ok) status, err := suite.typeconverter.ASStatusToStatus(rep) @@ -418,7 +419,7 @@ func (suite *ASToInternalTestSuite) TestParseStatusWithMention() { first := obj.Begin() assert.NotNil(suite.T(), first) - rep, ok := first.GetType().(typeutils.Statusable) + rep, ok := first.GetType().(ap.Statusable) assert.True(suite.T(), ok) status, err := suite.typeconverter.ASStatusToStatus(rep) diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 57c2a1f6d..10d9a0f18 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -19,7 +19,10 @@ package typeutils import ( + "net/url" + "github.com/go-fed/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -99,17 +102,17 @@ type TypeConverter interface { // If update is false, and the account is already known in the database, then the existing account entry will be returned. // If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account // will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile. - ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error) + ASRepresentationToAccount(accountable ap.Accountable, update bool) (*gtsmodel.Account, error) // ASStatus converts a remote activitystreams 'status' representation into a gts model status. - ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error) + ASStatusToStatus(statusable ap.Statusable) (*gtsmodel.Status, error) // ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request. - ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error) + ASFollowToFollowRequest(followable ap.Followable) (*gtsmodel.FollowRequest, error) // ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow. - ASFollowToFollow(followable Followable) (*gtsmodel.Follow, error) + ASFollowToFollow(followable ap.Followable) (*gtsmodel.Follow, error) // ASLikeToFave converts a remote activitystreams 'like' representation into a gts model status fave. - ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error) + ASLikeToFave(likeable ap.Likeable) (*gtsmodel.StatusFave, error) // ASBlockToBlock converts a remote activity streams 'block' representation into a gts model block. - ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) + ASBlockToBlock(blockable ap.Blockable) (*gtsmodel.Block, error) // ASAnnounceToStatus converts an activitystreams 'announce' into a status. // // The returned bool indicates whether this status is new (true) or not new (false). @@ -122,7 +125,7 @@ type TypeConverter interface { // This is useful when multiple users on an instance might receive the same boost, and we only want to process the boost once. // // NOTE -- this is different from one status being boosted multiple times! In this case, new boosts should indeed be created. - ASAnnounceToStatus(announceable Announceable) (status *gtsmodel.Status, new bool, err error) + ASAnnounceToStatus(announceable ap.Announceable) (status *gtsmodel.Status, new bool, err error) /* INTERNAL (gts) MODEL TO ACTIVITYSTREAMS MODEL @@ -150,7 +153,10 @@ type TypeConverter interface { BoostToAS(boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) (vocab.ActivityStreamsAnnounce, error) // BlockToAS converts a gts model block into an activityStreams BLOCK, suitable for federation. BlockToAS(block *gtsmodel.Block) (vocab.ActivityStreamsBlock, error) - + // StatusToASRepliesCollection converts a gts model status into an activityStreams REPLIES collection. + StatusToASRepliesCollection(status *gtsmodel.Status, onlyOtherAccounts bool) (vocab.ActivityStreamsCollection, error) + // StatusURIsToASRepliesPage returns a collection page with appropriate next/part of pagination. + StatusURIsToASRepliesPage(status *gtsmodel.Status, onlyOtherAccounts bool, minID string, replies map[string]*url.URL) (vocab.ActivityStreamsCollectionPage, error) /* INTERNAL (gts) MODEL TO INTERNAL MODEL */ diff --git a/internal/typeutils/converter_test.go b/internal/typeutils/converter_test.go index b2272f50c..c104ab06c 100644 --- a/internal/typeutils/converter_test.go +++ b/internal/typeutils/converter_test.go @@ -21,6 +21,7 @@ package typeutils_test import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -34,7 +35,7 @@ type ConverterStandardTestSuite struct { db db.DB log *logrus.Logger accounts map[string]*gtsmodel.Account - people map[string]typeutils.Accountable + people map[string]ap.Accountable typeconverter typeutils.TypeConverter } diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go index b24b07e13..333f131d4 100644 --- a/internal/typeutils/internaltoas.go +++ b/internal/typeutils/internaltoas.go @@ -27,6 +27,7 @@ import ( "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -505,7 +506,14 @@ func (c *converter) StatusToAS(s *gtsmodel.Status) (vocab.ActivityStreamsNote, e status.SetActivityStreamsAttachment(attachmentProp) // replies - // TODO + repliesCollection, err := c.StatusToASRepliesCollection(s, false) + if err != nil { + return nil, fmt.Errorf("error creating repliesCollection: %s", err) + } + + repliesProp := streams.NewActivityStreamsRepliesProperty() + repliesProp.SetActivityStreamsCollection(repliesCollection) + status.SetActivityStreamsReplies(repliesProp) return status, nil } @@ -850,3 +858,138 @@ func (c *converter) BlockToAS(b *gtsmodel.Block) (vocab.ActivityStreamsBlock, er return block, nil } + +/* + the goal is to end up with something like this: + + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies", + "type": "Collection", + "first": { + "id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?page=true", + "type": "CollectionPage", + "next": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?only_other_accounts=true&page=true", + "partOf": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies", + "items": [] + } + } +*/ +func (c *converter) StatusToASRepliesCollection(status *gtsmodel.Status, onlyOtherAccounts bool) (vocab.ActivityStreamsCollection, error) { + collectionID := fmt.Sprintf("%s/replies", status.URI) + collectionIDURI, err := url.Parse(collectionID) + if err != nil { + return nil, err + } + + collection := streams.NewActivityStreamsCollection() + + // collection.id + collectionIDProp := streams.NewJSONLDIdProperty() + collectionIDProp.SetIRI(collectionIDURI) + collection.SetJSONLDId(collectionIDProp) + + // first + first := streams.NewActivityStreamsFirstProperty() + firstPage := streams.NewActivityStreamsCollectionPage() + + // first.id + firstPageIDProp := streams.NewJSONLDIdProperty() + firstPageID, err := url.Parse(fmt.Sprintf("%s?page=true", collectionID)) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + firstPageIDProp.SetIRI(firstPageID) + firstPage.SetJSONLDId(firstPageIDProp) + + // first.next + nextProp := streams.NewActivityStreamsNextProperty() + nextPropID, err := url.Parse(fmt.Sprintf("%s?only_other_accounts=%t&page=true", collectionID, onlyOtherAccounts)) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + nextProp.SetIRI(nextPropID) + firstPage.SetActivityStreamsNext(nextProp) + + // first.partOf + partOfProp := streams.NewActivityStreamsPartOfProperty() + partOfProp.SetIRI(collectionIDURI) + firstPage.SetActivityStreamsPartOf(partOfProp) + + first.SetActivityStreamsCollectionPage(firstPage) + + // collection.first + collection.SetActivityStreamsFirst(first) + + return collection, nil +} + +/* + the goal is to end up with something like this: + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?only_other_accounts=true&page=true", + "type": "CollectionPage", + "next": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?min_id=106720870266901180&only_other_accounts=true&page=true", + "partOf": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies", + "items": [ + "https://example.com/users/someone/statuses/106720752853216226", + "https://somewhere.online/users/eeeeeeeeeep/statuses/106720870163727231" + ] + } +*/ +func (c *converter) StatusURIsToASRepliesPage(status *gtsmodel.Status, onlyOtherAccounts bool, minID string, replies map[string]*url.URL) (vocab.ActivityStreamsCollectionPage, error) { + collectionID := fmt.Sprintf("%s/replies", status.URI) + + page := streams.NewActivityStreamsCollectionPage() + + // .id + pageIDProp := streams.NewJSONLDIdProperty() + pageIDString := fmt.Sprintf("%s?page=true&only_other_accounts=%t", collectionID, onlyOtherAccounts) + if minID != "" { + pageIDString = fmt.Sprintf("%s&min_id=%s", pageIDString, minID) + } + + pageID, err := url.Parse(pageIDString) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + pageIDProp.SetIRI(pageID) + page.SetJSONLDId(pageIDProp) + + // .partOf + collectionIDURI, err := url.Parse(collectionID) + if err != nil { + return nil, err + } + partOfProp := streams.NewActivityStreamsPartOfProperty() + partOfProp.SetIRI(collectionIDURI) + page.SetActivityStreamsPartOf(partOfProp) + + // .items + items := streams.NewActivityStreamsItemsProperty() + var highestID string + for k, v := range replies { + items.AppendIRI(v) + if k > highestID { + highestID = k + } + } + page.SetActivityStreamsItems(items) + + // .next + nextProp := streams.NewActivityStreamsNextProperty() + nextPropIDString := fmt.Sprintf("%s?only_other_accounts=%t&page=true", collectionID, onlyOtherAccounts) + if highestID != "" { + nextPropIDString = fmt.Sprintf("%s&min_id=%s", nextPropIDString, highestID) + } + + nextPropID, err := url.Parse(nextPropIDString) + if err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + nextProp.SetIRI(nextPropID) + page.SetActivityStreamsNext(nextProp) + + return page, nil +} diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go index 8eb827e35..caa56ce0d 100644 --- a/internal/typeutils/internaltoas_test.go +++ b/internal/typeutils/internaltoas_test.go @@ -47,7 +47,7 @@ func (suite *InternalToASTestSuite) SetupSuite() { } func (suite *InternalToASTestSuite) SetupTest() { - testrig.StandardDBSetup(suite.db) + testrig.StandardDBSetup(suite.db, nil) } // TearDownTest drops tables to make sure there's no data in the db diff --git a/testrig/db.go b/testrig/db.go index 01cf93934..fe38c3164 100644 --- a/testrig/db.go +++ b/testrig/db.go @@ -65,7 +65,14 @@ func NewTestDB() db.DB { } // StandardDBSetup populates a given db with all the necessary tables/models for perfoming tests. -func StandardDBSetup(db db.DB) { +// +// The accounts parameter is provided in case the db should be populated with a certain set of accounts. +// If accounts is nil, then the standard test accounts will be used. +// +// When testing http signatures, you should pass into this function the same accounts map that you generated +// signatures with, otherwise this function will randomly generate new keys for accounts and signature +// verification will fail. +func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) { for _, m := range testModels { if err := db.CreateTable(m); err != nil { panic(err) @@ -96,9 +103,17 @@ func StandardDBSetup(db db.DB) { } } - for _, v := range NewTestAccounts() { - if err := db.Put(v); err != nil { - panic(err) + if accounts == nil { + for _, v := range NewTestAccounts() { + if err := db.Put(v); err != nil { + panic(err) + } + } + } else { + for _, v := range accounts { + if err := db.Put(v); err != nil { + panic(err) + } } } diff --git a/testrig/testmodels.go b/testrig/testmodels.go index 1934170d2..da5cbe7af 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -36,9 +36,9 @@ import ( "github.com/go-fed/activity/pub" "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/oauth" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) // NewTestTokens returns a map of tokens keyed according to which account the token belongs to. @@ -443,9 +443,9 @@ func NewTestAccounts() map[string]*gtsmodel.Account { FeaturedCollectionURI: "http://fossbros-anonymous.io/users/foss_satan/collections/featured", ActorType: gtsmodel.ActivityStreamsPerson, AlsoKnownAs: "", - PrivateKey: nil, + PrivateKey: &rsa.PrivateKey{}, PublicKey: &rsa.PublicKey{}, - PublicKeyURI: "http://fossbros-anonymous.io/users/foss_satan#main-key", + PublicKeyURI: "http://fossbros-anonymous.io/users/foss_satan/main-key", SensitizedAt: time.Time{}, SilencedAt: time.Time{}, SuspendedAt: time.Time{}, @@ -1033,6 +1033,32 @@ func NewTestStatuses() map[string]*gtsmodel.Status { }, ActivityStreamsType: gtsmodel.ActivityStreamsNote, }, + "local_account_2_status_5": { + ID: "01FCQSQ667XHJ9AV9T27SJJSX5", + URI: "http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5", + URL: "http://localhost:8080/@1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5", + Content: "🐢 hi zork! 🐢", + CreatedAt: time.Now().Add(-1 * time.Minute), + UpdatedAt: time.Now().Add(-1 * time.Minute), + Local: true, + AccountID: "01F8MH5NBDF2MV7CTC4Q5128HF", + InReplyToID: "01F8MHAMCHF6Y650WCRSCP4WMY", + InReplyToAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + InReplyToURI: "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY", + BoostOfID: "", + ContentWarning: "", + Visibility: gtsmodel.VisibilityPublic, + Sensitive: false, + Language: "en", + CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ", + VisibilityAdvanced: >smodel.VisibilityAdvanced{ + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + }, + ActivityStreamsType: gtsmodel.ActivityStreamsNote, + }, } } @@ -1155,14 +1181,14 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit } // NewTestFediPeople returns a bunch of activity pub Person representations for testing converters and so on. -func NewTestFediPeople() map[string]typeutils.Accountable { +func NewTestFediPeople() map[string]ap.Accountable { newPerson1Priv, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { panic(err) } newPerson1Pub := &newPerson1Priv.PublicKey - return map[string]typeutils.Accountable{ + return map[string]ap.Accountable{ "new_person_1": newPerson( URLMustParse("https://unknown-instance.com/users/brand_new_person"), URLMustParse("https://unknown-instance.com/users/brand_new_person/following"), @@ -1187,13 +1213,47 @@ func NewTestFediPeople() map[string]typeutils.Accountable { // NewTestDereferenceRequests returns a map of incoming dereference requests, with their signatures. func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[string]ActivityWithSignature { - sig, digest, date := getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, URLMustParse(accounts["local_account_1"].URI)) + var sig, digest, date string + var target *url.URL + statuses := NewTestStatuses() + + target = URLMustParse(accounts["local_account_1"].URI) + sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) + fossSatanDereferenceZork := ActivityWithSignature{ + SignatureHeader: sig, + DigestHeader: digest, + DateHeader: date, + } + + target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies") + sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) + fossSatanDereferenceLocalAccount1Status1Replies := ActivityWithSignature{ + SignatureHeader: sig, + DigestHeader: digest, + DateHeader: date, + } + + target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true") + sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) + fossSatanDereferenceLocalAccount1Status1RepliesNext := ActivityWithSignature{ + SignatureHeader: sig, + DigestHeader: digest, + DateHeader: date, + } + + target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5") + sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) + fossSatanDereferenceLocalAccount1Status1RepliesLast := ActivityWithSignature{ + SignatureHeader: sig, + DigestHeader: digest, + DateHeader: date, + } + return map[string]ActivityWithSignature{ - "foss_satan_dereference_zork": { - SignatureHeader: sig, - DigestHeader: digest, - DateHeader: date, - }, + "foss_satan_dereference_zork": fossSatanDereferenceZork, + "foss_satan_dereference_local_account_1_status_1_replies": fossSatanDereferenceLocalAccount1Status1Replies, + "foss_satan_dereference_local_account_1_status_1_replies_next": fossSatanDereferenceLocalAccount1Status1RepliesNext, + "foss_satan_dereference_local_account_1_status_1_replies_last": fossSatanDereferenceLocalAccount1Status1RepliesLast, } } @@ -1215,7 +1275,7 @@ func getSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry } // use the client to create a new transport - c := NewTestTransportController(client) + c := NewTestTransportController(client, NewTestDB()) tp, err := c.NewTransport(pubKeyID, privkey) if err != nil { panic(err) @@ -1247,7 +1307,6 @@ func getSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest client := &mockHTTPClient{ do: func(req *http.Request) (*http.Response, error) { signatureHeader = req.Header.Get("Signature") - digestHeader = req.Header.Get("Digest") dateHeader = req.Header.Get("Date") r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out return &http.Response{ @@ -1258,7 +1317,7 @@ func getSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest } // use the client to create a new transport - c := NewTestTransportController(client) + c := NewTestTransportController(client, NewTestDB()) tp, err := c.NewTransport(pubKeyID, privkey) if err != nil { panic(err) @@ -1290,7 +1349,7 @@ func newPerson( avatarURL *url.URL, avatarContentType string, headerURL *url.URL, - headerContentType string) typeutils.Accountable { + headerContentType string) ap.Accountable { person := streams.NewActivityStreamsPerson() // id should be the activitypub URI of this user diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go index f2b5b93f7..ec591a9b1 100644 --- a/testrig/transportcontroller.go +++ b/testrig/transportcontroller.go @@ -24,6 +24,7 @@ import ( "net/http" "github.com/go-fed/activity/pub" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/transport" ) @@ -37,8 +38,8 @@ import ( // Unlike the other test interfaces provided in this package, you'll probably want to call this function // PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular) // basis. -func NewTestTransportController(client pub.HttpClient) transport.Controller { - return transport.NewController(NewTestConfig(), &federation.Clock{}, client, NewTestLog()) +func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller { + return transport.NewController(NewTestConfig(), db, &federation.Clock{}, client, NewTestLog()) } // NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,