Compare commits

..
1 Commits
Author SHA1 Message Date
marc d7713db58b Dockerfile
Service CI / build (push) Canceled after 0s
2021-12-23 14:03:11 +01:00
177 changed files with 5154 additions and 6820 deletions
+2 -2
View File
@@ -11,8 +11,8 @@ jobs:
- name: Set up JDK 11
uses: actions/setup-java@3bc31aaf88e8fc94dc1e632d48af61be5ca8721c
with:
distribution: 'temurin'
java-version: 17
distribution: 'adopt'
java-version: 11
cache: 'maven'
- name: Build with Maven
run: mvn -e -B verify
+1 -1
View File
@@ -1,4 +1,4 @@
# Note that the implementation of the abusive message filter is private; internal
# Note that the implmentation of the abusive message filter is private; internal
# developers will need to override this URL with:
#
# ```
+6
View File
@@ -0,0 +1,6 @@
FROM maven
COPY . /app
WORKDIR /app
RUN mvn install
+5 -18
View File
@@ -34,7 +34,7 @@
<commons-codec.version>1.15</commons-codec.version>
<commons-csv.version>1.8</commons-csv.version>
<commons-io.version>2.9.0</commons-io.version>
<dropwizard.version>2.0.25</dropwizard.version>
<dropwizard.version>2.0.22</dropwizard.version>
<dropwizard-metrics-datadog.version>1.1.13</dropwizard-metrics-datadog.version>
<gson.version>2.8.8</gson.version>
<guava.version>30.1.1-jre</guava.version>
@@ -44,7 +44,7 @@
<libphonenumber.version>8.12.33</libphonenumber.version>
<logstash.logback.version>6.6</logstash.logback.version>
<micrometer.version>1.5.3</micrometer.version>
<mockito.version>4.0.0</mockito.version>
<mockito.version>3.11.1</mockito.version>
<netty.version>4.1.65.Final</netty.version>
<netty.tcnative-boringssl-static.version>2.0.39.Final</netty.tcnative-boringssl-static.version>
<opentest4j.version>1.2.0</opentest4j.version>
@@ -253,19 +253,6 @@
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
<dependency>
<groupId>org.signal</groupId>
<artifactId>embedded-redis</artifactId>
<version>0.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.zonky.test.postgres</groupId>
<artifactId>embedded-postgres-binaries-bom</artifactId>
<version>11.13.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
@@ -350,7 +337,6 @@
<version>0.6.1</version>
<configuration>
<protocArtifact>com.google.protobuf:protoc:3.18.0:exe:${os.detected.classifier}</protocArtifact>
<checkStaleness>true</checkStaleness>
</configuration>
<executions>
<execution>
@@ -367,7 +353,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<release>17</release>
<source>11</source>
<target>11</target>
</configuration>
</plugin>
@@ -431,7 +418,7 @@
<rules>
<dependencyConvergence/>
<requireMavenVersion>
<version>3.8.3</version>
<version>3.0.0</version>
</requireMavenVersion>
</rules>
</configuration>
+167 -277
View File
@@ -1,71 +1,59 @@
# Example, relatively minimal, configuration that passes validation (see `io.dropwizard.cli.CheckCommand`)
#
# `unset` values will need to be set to work properly.
# Most other values are technically valid for a local/demonstration environment, but are probably not production-ready.
stripe:
apiKey: unset
idempotencyKeyGenerator: abcdefg12345678= # base64 for creating request idempotency hash
boostDescription: >
Example
apiKey:
idempotencyKeyGenerator:
dynamoDbClientConfiguration:
region: us-west-2 # AWS Region
region: # AWS Region
dynamoDbTables:
issuedReceipts:
tableName: Example_IssuedReceipts
expiration: P30D # Duration of time until rows expire
generator: abcdefg12345678= # random base64-encoded binary sequence
tableName: # DDB Table Name
expiration: # Duration of time until rows expire
generator: # random binary sequence
redeemedReceipts:
tableName: Example_RedeemedReceipts
expiration: P30D # Duration of time until rows expire
tableName: # DDB Table Name
expiration: # Duration of time until rows expire
subscriptions:
tableName: Example_Subscriptions
profiles:
tableName: Example_Profiles
remoteConfig:
tableName: Example_RemoteConfig
tableName: # DDB Table Name
twilio: # Twilio gateway configuration
accountId: unset
accountToken: unset
nanpaMessagingServiceSid: unset # Twilio SID for the messaging service to use for NANPA.
messagingServiceSid: unset # Twilio SID for the message service to use for non-NANPA.
verifyServiceSid: unset # Twilio SID for a Verify service
localDomain: example.com # Domain Twilio can connect back to for calls. Should be domain of your service.
accountId:
accountToken:
nanpaMessagingServiceSid: # Twilio SID for the messaging service to use for NANPA.
messagingServiceSid: # Twilio SID for the message service to use for non-NANPA.
verifyServiceSid: # Twilio SID for a Verify service
localDomain: # Domain Twilio can connect back to for calls. Should be domain of your service.
defaultClientVerificationTexts:
ios: example %1$s # Text to use for the verification message on iOS. Will be passed to String.format with the verification code as argument 1.
androidNg: example %1$s # Text to use for the verification message on android-ng client types. Will be passed to String.format with the verification code as argument 1.
android202001: example %1$s # Text to use for the verification message on android-2020-01 client types. Will be passed to String.format with the verification code as argument 1.
android202103: example %1$s # Text to use for the verification message on android-2021-03 client types. Will be passed to String.format with the verification code as argument 1.
generic: example %1$s # Text to use when the client type is unrecognized. Will be passed to String.format with the verification code as argument 1.
ios: # Text to use for the verification message on iOS. Will be passed to String.format with the verification code as argument 1.
androidNg: # Text to use for the verification message on android-ng client types. Will be passed to String.format with the verification code as argument 1.
android202001: # Text to use for the verification message on android-2020-01 client types. Will be passed to String.format with the verification code as argument 1.
android202103: # Text to use for the verification message on android-2021-03 client types. Will be passed to String.format with the verification code as argument 1.
generic: # Text to use when the client type is unrecognized. Will be passed to String.format with the verification code as argument 1.
regionalClientVerificationTexts: # Map of country codes to custom texts
999: # example country code
ios: example %1$s # all keys from defaultClientVerificationTexts are required
androidNg: example %1$s
android202001: example %1$s
android202103: example %1$s
generic: example %1$s
androidAppHash: example # Hash appended to Android
verifyServiceFriendlyName: example # Service name used in template. Requires Twilio account rep to enable
ios:
# … all keys from defaultClientVerificationTexts are required
androidAppHash: # Hash appended to Android
verifyServiceFriendlyName: # Service name used in template. Requires Twilio account rep to enable
push:
queueSize: 1000 # Size of push pending queue
queueSize: # Size of push pending queue
turn: # TURN server configuration
secret: example # TURN server secret
secret: # TURN server secret
uris:
- stun:example.com:80
- stun:another.example.com:443
- turn:example.com:443?transport=udp
- turn:ya.example.com:80?transport=udp
- stun:yourdomain:80
- stun:yourdomain.com:443
- turn:yourdomain:443?transport=udp
- turn:etc.com:80?transport=udp
cacheCluster: # Redis server configuration for cache cluster
configurationUri: redis://redis.example.com:6379/
urls:
- redis://redis.example.com:6379/
clientPresenceCluster: # Redis server configuration for client presence cluster
configurationUri: redis://redis.example.com:6379/
urls:
- redis://redis.example.com:6379/
pubsub: # Redis server configuration for pubsub cluster
url: redis://redis.example.com:6379/
@@ -73,189 +61,148 @@ pubsub: # Redis server configuration for pubsub cluster
- redis://redis.example.com:6379/
pushSchedulerCluster: # Redis server configuration for push scheduler cluster
configurationUri: redis://redis.example.com:6379/
urls:
- redis://redis.example.com:6379/
rateLimitersCluster: # Redis server configuration for rate limiters cluster
configurationUri: redis://redis.example.com:6379/
urls:
- redis://redis.example.com:6379/
directory:
client: # Configuration for interfacing with Contact Discovery Service cluster
userAuthenticationTokenSharedSecret: 00000f # hex-encoded secret shared with CDS used to generate auth tokens for Signal users
userAuthenticationTokenUserIdSecret: 00000f # hex-encoded secret shared among Signal-Servers to obscure user phone numbers from CDS
userAuthenticationTokenSharedSecret: # hex-encoded secret shared with CDS used to generate auth tokens for Signal users
userAuthenticationTokenUserIdSecret: # hex-encoded secret shared among Signal-Servers to obscure user phone numbers from CDS
sqs:
accessKey: test # AWS SQS accessKey
accessSecret: test # AWS SQS accessSecret
queueUrls: # AWS SQS queue urls
accessKey: # AWS SQS accessKey
accessSecret: # AWS SQS accessSecret
queueUrls: # AWS SQS queue urls
- https://sqs.example.com/directory.fifo
server: # One or more CDS servers
- replicationName: example # CDS replication name
replicationUrl: cds.example.com # CDS replication endpoint base url
replicationPassword: example # CDS replication endpoint password
replicationCaCertificate: | # CDS replication endpoint TLS certificate trust root
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
directoryV2:
client: # Configuration for interfacing with Contact Discovery Service v2 cluster
userAuthenticationTokenSharedSecret: abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG= # base64-encoded secret shared with CDS to generate auth tokens for Signal users
- replicationName: # CDS replication name
replicationUrl: # CDS replication endpoint base url
replicationPassword: # CDS replication endpoint password
replicationCaCertificate: # CDS replication endpoint TLS certificate trust root
messageCache: # Redis server configuration for message store cache
persistDelayMinutes: 1
persistDelayMinutes:
cluster:
configurationUri: redis://redis.example.com:6379/
urls:
- redis://redis.example.com:6379/
metricsCluster:
configurationUri: redis://redis.example.com:6379/
urls:
- redis://redis.example.com:6379/
messageDynamoDb: # DynamoDB table configuration
region: us-west-2
tableName: Example_Messages
region:
tableName:
keysDynamoDb: # DynamoDB table configuration
region: us-west-2
tableName: Example_PreKeys
region:
tableName:
accountsDynamoDb: # DynamoDB table configuration
region: us-west-2
tableName: Example_Accounts
phoneNumberTableName: Example_Accounts_PhoneNumbers
phoneNumberIdentifierTableName: Example_Accounts_PhoneNumberIdentifiers
usernamesTableName: Example_Accounts_Usernames
region:
tableName:
phoneNumberTableName:
deletedAccountsDynamoDb: # DynamoDb table configuration
region: us-west-2
tableName: Example_DeletedAccounts
needsReconciliationIndexName: NeedsReconciliation
region:
tableName:
needsReconciliationIndexName:
deletedAccountsLockDynamoDb: # DynamoDb table configuration
region: us-west-2
tableName: Example_DeletedAccountsLock
region:
tableName:
redeemedReceiptsDynamoDb: # DynamoDB table configuration
region:
tableName:
expirationTime: # ISO8601 Duration
migrationDeletedAccountsDynamoDb: # DynamoDB table configuration
region:
tableName:
migrationRetryAccountsDynamoDb: # DynamoDB table configuration
region:
tableName:
pendingAccountsDynamoDb: # DynamoDB table configuration
region: us-west-2
tableName: Example_PendingAccounts
region:
tableName:
pendingDevicesDynamoDb: # DynamoDB table configuration
region: us-west-2
tableName: Example_PendingDevices
reservedUsernamesDynamoDb: # DynamoDB table configuration
region: us-west-2
tableName: Example_ReservedUsernames
phoneNumberIdentifiersDynamoDb: # DynamoDB table configuration
region: us-west-2
tableName: Example_PhoneNumberIdentifiers
region:
tableName:
pushChallengeDynamoDb: # DynamoDB table configuration
region: us-west-2
tableName: Example_PushChallenges
region:
tableName:
reportMessageDynamoDb: # DynamoDB table configuration
region: us-west-2
tableName: Example_ReportedMessages
region:
tableName:
awsAttachments: # AWS S3 configuration
accessKey: test
accessSecret: test
bucket: aws-attachments
region: us-west-2
accessKey:
accessSecret:
bucket:
region:
gcpAttachments: # GCP Storage configuration
domain: example.com
email: user@example.cocm
maxSizeInBytes: 1024
domain:
email:
maxSizeInBytes:
pathPrefix:
rsaSigningKey: |
-----BEGIN PRIVATE KEY-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAA
-----END PRIVATE KEY-----
rsaSigningKey:
abuseDatabase: # Postgresql database configuration
driverClass: org.postgresql.Driver
user: example
password: password
url: jdbc:postgresql://example.com:5432/abusedb
user:
password:
url:
accountsDatabase: # Postgresql database configuration
driverClass: org.postgresql.Driver
user:
password:
url:
accountDatabaseCrawler:
chunkSize: 10 # accounts per run
chunkIntervalMs: 60000 # time per run
chunkSize: # accounts per run
chunkIntervalMs: # time per run
dynamoDbMigrationCrawler:
chunkSize: # accounts per run
chunkIntervalMs: # time per run
apn: # Apple Push Notifications configuration
sandbox: true
bundleId: com.example.textsecuregcm
keyId: unset
teamId: unset
signingKey: |
-----BEGIN PRIVATE KEY-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAA
-----END PRIVATE KEY-----
bundleId:
keyId:
teamId:
signingKey:
gcm: # GCM Configuration
senderId: 123456789
apiKey: unset
senderId:
apiKey:
cdn:
accessKey: test # AWS Access Key ID
accessSecret: test # AWS Access Secret
bucket: cdn # S3 Bucket name
region: us-west-2 # AWS region
accessKey: # AWS Access Key ID
accessSecret: # AWS Access Secret
bucket: # S3 Bucket name
region: # AWS region
datadog:
apiKey: unset
environment: dev
apiKey:
environment:
unidentifiedDelivery:
certificate: ABCD1234
privateKey: ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789AAAAAAA
expiresDays: 7
certificate:
privateKey:
expiresDays:
voiceVerification:
url: https://cdn-ca.signal.org/verification/
@@ -263,74 +210,33 @@ voiceVerification:
- en
recaptcha:
secret: unset
secret:
recaptchaV2:
siteKey: unset
scoreFloor: 1.0
projectPath: projects/example
credentialConfigurationJson: "{ }" # service account configuration for backend authentication
siteKey:
scoreFloor:
projectPath:
credentialConfigurationJson:
storageService:
uri: storage.example.com
userAuthenticationTokenSharedSecret: 00000f
storageCaCertificate: |
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
uri:
userAuthenticationTokenSharedSecret:
storageCaCertificate:
backupService:
uri: backup.example.com
userAuthenticationTokenSharedSecret: 00000f
backupCaCertificate: |
-----BEGIN CERTIFICATE-----
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
AAAAAAAAAAAAAAAAAAAA
-----END CERTIFICATE-----
uri:
userAuthenticationTokenSharedSecret:
backupCaCertificate:
zkConfig:
serverPublic: ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyz
serverSecret: ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789+abcdefghijklmnopqrstuvwxyzAA==
serverPublic:
serverSecret:
enabled:
appConfig:
application: example
environment: example
configuration: example
application:
environment:
configuration:
remoteConfig:
authorizedTokens:
@@ -339,64 +245,48 @@ remoteConfig:
- # ...
- # Nth authorized token
globalConfig: # keys and values that are given to clients on GET /v1/config
EXAMPLE_KEY: VALUE
paymentsService:
userAuthenticationTokenSharedSecret: 0000000f0000000f0000000f0000000f0000000f0000000f0000000f0000000f # hex-encoded 32-byte secret shared with MobileCoin services used to generate auth tokens for Signal users
fixerApiKey: unset
userAuthenticationTokenSharedSecret: # hex-encoded 32-byte secret shared with MobileCoin services used to generate auth tokens for Signal users
fixerApiKey:
paymentCurrencies:
# list of symbols for supported currencies
- MOB
-
torExitNodeList:
s3Region:
s3Bucket:
objectKey:
maxSize:
asnTable:
s3Region:
s3Bucket:
objectKey:
maxSize:
donation:
uri: donation.example.com # value
uri: # value
supportedCurrencies:
- # 1st supported currency
- # 2nd supported currency
- # ...
- # Nth supported currency
circuitBreaker:
failureRateThreshold: # value
ringBufferSizeInHalfOpenState: # value
ringBufferSizeInClosedState: # value
waitDurationInOpenStateInSeconds: # value
retry:
maxAttempts: # value
waitDuration: # value
badges:
badges:
- id: TEST
imageUrl: https://example.com/test-badge
category: other
sprites: # exactly 6
- sprite-1.png
- sprite-2.png
- sprite-3.png
- sprite-4.png
- sprite-5.png
- sprite-6.png
svg: example.svg
svgs:
- light: example-light.svg
dark: example-dark.svg
badgeIdsEnabledForAll:
- TEST
receiptLevels:
'1': TEST
subscription: # configuration for Stripe subscriptions
badgeGracePeriod: P15D
levels:
500:
badge: EXAMPLE
prices:
# list of ISO 4217 currency codes and amounts for the given badge level
xts:
amount: '10'
id: price_example # stripe ID
boost:
level: 1
expiration: P90D
badge: EXAMPLE
currencies:
# ISO 4217 currency codes and amounts in those currencies
xts:
- '1'
- '2'
- '4'
- '8'
- '20'
- '40'
+5 -26
View File
@@ -55,7 +55,7 @@
<dependency>
<groupId>org.signal</groupId>
<artifactId>zkgroup-java</artifactId>
<version>0.9.0</version>
<version>0.8.2</version>
</dependency>
<dependency>
<groupId>org.whispersystems</groupId>
@@ -382,9 +382,9 @@
</dependency>
<dependency>
<groupId>io.zonky.test</groupId>
<artifactId>embedded-postgres</artifactId>
<version>1.3.1</version>
<groupId>com.opentable.components</groupId>
<artifactId>otj-pg-embedded</artifactId>
<version>0.13.3</version>
<scope>test</scope>
</dependency>
@@ -420,6 +420,7 @@
<dependency>
<groupId>org.signal</groupId>
<artifactId>embedded-redis</artifactId>
<version>0.8.1</version>
<scope>test</scope>
</dependency>
@@ -592,28 +593,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<id>check-all-service-config</id>
<phase>verify</phase>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions>
<configuration>
<mainClass>org.whispersystems.textsecuregcm.CheckServiceConfigurations</mainClass>
<classpathScope>test</classpathScope>
<arguments>
<argument>${project.basedir}/config</argument>
</arguments>
</configuration>
</plugin>
</plugins>
</build>
</project>
@@ -15,6 +15,7 @@ import javax.validation.Valid;
import javax.validation.constraints.NotNull;
import org.whispersystems.textsecuregcm.configuration.AbusiveMessageFilterConfiguration;
import org.whispersystems.textsecuregcm.configuration.AccountDatabaseCrawlerConfiguration;
import org.whispersystems.textsecuregcm.configuration.AccountsDatabaseConfiguration;
import org.whispersystems.textsecuregcm.configuration.AccountsDynamoDbConfiguration;
import org.whispersystems.textsecuregcm.configuration.ApnConfiguration;
import org.whispersystems.textsecuregcm.configuration.AppConfigConfiguration;
@@ -26,7 +27,6 @@ import org.whispersystems.textsecuregcm.configuration.DatabaseConfiguration;
import org.whispersystems.textsecuregcm.configuration.DatadogConfiguration;
import org.whispersystems.textsecuregcm.configuration.DeletedAccountsDynamoDbConfiguration;
import org.whispersystems.textsecuregcm.configuration.DirectoryConfiguration;
import org.whispersystems.textsecuregcm.configuration.DirectoryV2Configuration;
import org.whispersystems.textsecuregcm.configuration.DonationConfiguration;
import org.whispersystems.textsecuregcm.configuration.DynamoDbClientConfiguration;
import org.whispersystems.textsecuregcm.configuration.DynamoDbConfiguration;
@@ -125,11 +125,6 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private DirectoryConfiguration directory;
@NotNull
@Valid
@JsonProperty
private DirectoryV2Configuration directoryV2;
@NotNull
@Valid
@JsonProperty
@@ -170,11 +165,6 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private AccountsDynamoDbConfiguration accountsDynamoDb;
@Valid
@NotNull
@JsonProperty
private DynamoDbConfiguration phoneNumberIdentifiersDynamoDb;
@Valid
@NotNull
@JsonProperty
@@ -205,11 +195,6 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private DynamoDbConfiguration pendingDevicesDynamoDb;
@Valid
@NotNull
@JsonProperty
private DynamoDbConfiguration reservedUsernamesDynamoDb;
@Valid
@NotNull
@JsonProperty
@@ -225,6 +210,11 @@ public class WhisperServerConfiguration extends Configuration {
@JsonProperty
private List<MaxDeviceConfiguration> maxDevices = new LinkedList<>();
@Valid
@NotNull
@JsonProperty
private AccountsDatabaseConfiguration accountsDatabase;
@Valid
@NotNull
@JsonProperty
@@ -400,10 +390,6 @@ public class WhisperServerConfiguration extends Configuration {
return directory;
}
public DirectoryV2Configuration getDirectoryV2Configuration() {
return directoryV2;
}
public SecureStorageServiceConfiguration getSecureStorageServiceConfiguration() {
return storageService;
}
@@ -440,10 +426,6 @@ public class WhisperServerConfiguration extends Configuration {
return accountsDynamoDb;
}
public DynamoDbConfiguration getPhoneNumberIdentifiersDynamoDbConfiguration() {
return phoneNumberIdentifiersDynamoDb;
}
public DeletedAccountsDynamoDbConfiguration getDeletedAccountsDynamoDbConfiguration() {
return deletedAccountsDynamoDb;
}
@@ -456,6 +438,10 @@ public class WhisperServerConfiguration extends Configuration {
return abuseDatabase;
}
public AccountsDatabaseConfiguration getAccountsDatabaseConfiguration() {
return accountsDatabase;
}
public RateLimitsConfiguration getLimitsConfiguration() {
return limits;
}
@@ -546,10 +532,6 @@ public class WhisperServerConfiguration extends Configuration {
return pendingDevicesDynamoDb;
}
public DynamoDbConfiguration getReservedUsernamesDynamoDbConfiguration() {
return reservedUsernamesDynamoDb;
}
public DonationConfiguration getDonationConfiguration() {
return donation;
}
@@ -24,6 +24,7 @@ import io.dropwizard.auth.PolymorphicAuthDynamicFeature;
import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
import io.dropwizard.auth.basic.BasicCredentialAuthFilter;
import io.dropwizard.auth.basic.BasicCredentials;
import io.dropwizard.db.DataSourceFactory;
import io.dropwizard.db.PooledDataSourceFactory;
import io.dropwizard.jdbi3.JdbiFactory;
import io.dropwizard.setup.Bootstrap;
@@ -52,6 +53,7 @@ import java.util.concurrent.TimeUnit;
import javax.servlet.DispatcherType;
import javax.servlet.FilterRegistration;
import javax.servlet.ServletRegistration;
import javax.ws.rs.container.DynamicFeature;
import org.eclipse.jetty.servlets.CrossOriginFilter;
import org.glassfish.jersey.server.ServerProperties;
import org.jdbi.v3.core.Jdbi;
@@ -66,7 +68,6 @@ import org.slf4j.LoggerFactory;
import org.whispersystems.dispatch.DispatchManager;
import org.whispersystems.textsecuregcm.abuse.AbusiveMessageFilter;
import org.whispersystems.textsecuregcm.abuse.FilterAbusiveMessages;
import org.whispersystems.textsecuregcm.abuse.RateLimitChallengeListener;
import org.whispersystems.textsecuregcm.auth.AccountAuthenticator;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
import org.whispersystems.textsecuregcm.auth.CertificateGenerator;
@@ -79,7 +80,6 @@ import org.whispersystems.textsecuregcm.badges.ConfiguredProfileBadgeConverter;
import org.whispersystems.textsecuregcm.badges.ResourceBundleLevelTranslator;
import org.whispersystems.textsecuregcm.configuration.DirectoryServerConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.controllers.AcceptNumericOnlineFlagRequestFilter;
import org.whispersystems.textsecuregcm.controllers.AccountController;
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV1;
import org.whispersystems.textsecuregcm.controllers.AttachmentControllerV2;
@@ -88,7 +88,6 @@ import org.whispersystems.textsecuregcm.controllers.CertificateController;
import org.whispersystems.textsecuregcm.controllers.ChallengeController;
import org.whispersystems.textsecuregcm.controllers.DeviceController;
import org.whispersystems.textsecuregcm.controllers.DirectoryController;
import org.whispersystems.textsecuregcm.controllers.DirectoryV2Controller;
import org.whispersystems.textsecuregcm.controllers.DonationController;
import org.whispersystems.textsecuregcm.controllers.KeepAliveController;
import org.whispersystems.textsecuregcm.controllers.KeysController;
@@ -109,13 +108,13 @@ import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
import org.whispersystems.textsecuregcm.filters.ContentLengthFilter;
import org.whispersystems.textsecuregcm.filters.RemoteDeprecationFilter;
import org.whispersystems.textsecuregcm.filters.TimestampResponseFilter;
import org.whispersystems.textsecuregcm.limits.DynamicRateLimiters;
import org.whispersystems.textsecuregcm.limits.PreKeyRateLimiter;
import org.whispersystems.textsecuregcm.limits.PushChallengeManager;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeOptionManager;
import org.whispersystems.textsecuregcm.limits.RateLimitResetMetricsManager;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.limits.UnsealedSenderRateLimiter;
import org.whispersystems.textsecuregcm.liquibase.NameableMigrationsBundle;
import org.whispersystems.textsecuregcm.mappers.CompletionExceptionMapper;
import org.whispersystems.textsecuregcm.mappers.DeviceLimitExceededExceptionMapper;
import org.whispersystems.textsecuregcm.mappers.IOExceptionMapper;
import org.whispersystems.textsecuregcm.mappers.ImpossiblePhoneNumberExceptionMapper;
@@ -180,13 +179,12 @@ import org.whispersystems.textsecuregcm.storage.DirectoryReconciliationClient;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.storage.FaultTolerantDatabase;
import org.whispersystems.textsecuregcm.storage.IssuedReceiptsManager;
import org.whispersystems.textsecuregcm.storage.Keys;
import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagePersister;
import org.whispersystems.textsecuregcm.storage.MessagesCache;
import org.whispersystems.textsecuregcm.storage.MessagesDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.NonNormalizedAccountCrawlerListener;
import org.whispersystems.textsecuregcm.storage.PhoneNumberIdentifiers;
import org.whispersystems.textsecuregcm.storage.Profiles;
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
import org.whispersystems.textsecuregcm.storage.PubSubManager;
@@ -200,6 +198,8 @@ import org.whispersystems.textsecuregcm.storage.ReportMessageManager;
import org.whispersystems.textsecuregcm.storage.ReservedUsernames;
import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
import org.whispersystems.textsecuregcm.storage.SubscriptionManager;
import org.whispersystems.textsecuregcm.storage.Usernames;
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
import org.whispersystems.textsecuregcm.storage.VerificationCodeStore;
import org.whispersystems.textsecuregcm.stripe.StripeManager;
import org.whispersystems.textsecuregcm.util.Constants;
@@ -214,7 +214,6 @@ import org.whispersystems.textsecuregcm.websocket.WebSocketAccountAuthenticator;
import org.whispersystems.textsecuregcm.workers.CertificateCommand;
import org.whispersystems.textsecuregcm.workers.CheckDynamicConfigurationCommand;
import org.whispersystems.textsecuregcm.workers.DeleteUserCommand;
import org.whispersystems.textsecuregcm.workers.ReserveUsernameCommand;
import org.whispersystems.textsecuregcm.workers.ServerVersionCommand;
import org.whispersystems.textsecuregcm.workers.SetCrawlerAccelerationTask;
import org.whispersystems.textsecuregcm.workers.SetRequestLoggingEnabledTask;
@@ -241,7 +240,13 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
bootstrap.addCommand(new ServerVersionCommand());
bootstrap.addCommand(new CheckDynamicConfigurationCommand());
bootstrap.addCommand(new SetUserDiscoverabilityCommand());
bootstrap.addCommand(new ReserveUsernameCommand());
bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("accountdb", "accountsdb.xml") {
@Override
public DataSourceFactory getDataSourceFactory(WhisperServerConfiguration configuration) {
return configuration.getAccountsDatabaseConfiguration();
}
});
bootstrap.addBundle(new NameableMigrationsBundle<WhisperServerConfiguration>("abusedb", "abusedb.xml") {
@Override
@@ -305,18 +310,16 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
headerControlledResourceBundleLookup);
JdbiFactory jdbiFactory = new JdbiFactory(DefaultNameStrategy.CHECK_EMPTY);
Jdbi accountJdbi = jdbiFactory.build(environment, config.getAccountsDatabaseConfiguration(), "accountdb");
Jdbi abuseJdbi = jdbiFactory.build(environment, config.getAbuseDatabaseConfiguration(), "abusedb" );
FaultTolerantDatabase abuseDatabase = new FaultTolerantDatabase("abuse_database", abuseJdbi, config.getAbuseDatabaseConfiguration().getCircuitBreakerConfiguration());
FaultTolerantDatabase accountDatabase = new FaultTolerantDatabase("accounts_database", accountJdbi, config.getAccountsDatabaseConfiguration().getCircuitBreakerConfiguration());
FaultTolerantDatabase abuseDatabase = new FaultTolerantDatabase("abuse_database", abuseJdbi, config.getAbuseDatabaseConfiguration().getCircuitBreakerConfiguration());
DynamoDbAsyncClient dynamoDbAsyncClient = DynamoDbFromConfig.asyncClient(
config.getDynamoDbClientConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient dynamoDbClient = DynamoDbFromConfig.client(
config.getDynamoDbClientConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient messageDynamoDb = DynamoDbFromConfig.client(config.getMessageDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
@@ -326,13 +329,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
DynamoDbClient accountsDynamoDbClient = DynamoDbFromConfig.client(config.getAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient reservedUsernamesDynamoDbClient = DynamoDbFromConfig.client(config.getReservedUsernamesDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient phoneNumberIdentifiersDynamoDbClient =
DynamoDbFromConfig.client(config.getPhoneNumberIdentifiersDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
DynamoDbClient deletedAccountsDynamoDbClient = DynamoDbFromConfig.client(config.getDeletedAccountsDynamoDbConfiguration(),
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider.create());
@@ -368,21 +364,16 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
Accounts accounts = new Accounts(accountsDynamoDbClient,
config.getAccountsDynamoDbConfiguration().getTableName(),
config.getAccountsDynamoDbConfiguration().getPhoneNumberTableName(),
config.getAccountsDynamoDbConfiguration().getPhoneNumberIdentifierTableName(),
config.getAccountsDynamoDbConfiguration().getUsernamesTableName(),
config.getAccountsDynamoDbConfiguration().getScanPageSize());
PhoneNumberIdentifiers phoneNumberIdentifiers = new PhoneNumberIdentifiers(phoneNumberIdentifiersDynamoDbClient,
config.getPhoneNumberIdentifiersDynamoDbConfiguration().getTableName());
ReservedUsernames reservedUsernames = new ReservedUsernames(reservedUsernamesDynamoDbClient,
config.getReservedUsernamesDynamoDbConfiguration().getTableName());
Profiles profiles = new Profiles(dynamoDbClient, dynamoDbAsyncClient,
config.getDynamoDbTables().getProfiles().getTableName());
Keys keys = new Keys(preKeyDynamoDb, config.getKeysDynamoDbConfiguration().getTableName());
Usernames usernames = new Usernames(accountDatabase);
ReservedUsernames reservedUsernames = new ReservedUsernames(accountDatabase);
Profiles profiles = new Profiles(accountDatabase);
KeysDynamoDb keysDynamoDb = new KeysDynamoDb(preKeyDynamoDb, config.getKeysDynamoDbConfiguration().getTableName());
MessagesDynamoDb messagesDynamoDb = new MessagesDynamoDb(messageDynamoDb,
config.getMessageDynamoDbConfiguration().getTableName(),
config.getMessageDynamoDbConfiguration().getTimeToLive());
AbusiveHostRules abusiveHostRules = new AbusiveHostRules(abuseDatabase);
RemoteConfigs remoteConfigs = new RemoteConfigs(dynamoDbClient, config.getDynamoDbTables().getRemoteConfig().getTableName());
RemoteConfigs remoteConfigs = new RemoteConfigs(accountDatabase);
PushChallengeDynamoDb pushChallengeDynamoDb = new PushChallengeDynamoDb(pushChallengeDynamoDbClient, config.getPushChallengeDynamoDbConfiguration().getTableName());
ReportMessageDynamoDb reportMessageDynamoDb = new ReportMessageDynamoDb(reportMessageDynamoDbClient, config.getReportMessageDynamoDbConfiguration().getTableName(), config.getReportMessageConfiguration().getReportTtl());
VerificationCodeStore pendingAccounts = new VerificationCodeStore(pendingAccountsDynamoDbClient, config.getPendingAccountsDynamoDbConfiguration().getTableName());
@@ -391,15 +382,24 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
RedisClientFactory pubSubClientFactory = new RedisClientFactory("pubsub_cache", config.getPubsubCacheConfiguration().getUrl(), config.getPubsubCacheConfiguration().getReplicaUrls(), config.getPubsubCacheConfiguration().getCircuitBreakerConfiguration());
ReplicatedJedisPool pubsubClient = pubSubClientFactory.getRedisClientPool();
ClientResources redisClientResources = ClientResources.builder().build();
ConnectionEventLogger.logConnectionEvents(redisClientResources);
ClientResources generalCacheClientResources = ClientResources.builder().build();
ClientResources messageCacheClientResources = ClientResources.builder().build();
ClientResources presenceClientResources = ClientResources.builder().build();
ClientResources metricsCacheClientResources = ClientResources.builder().build();
ClientResources pushSchedulerCacheClientResources = ClientResources.builder().ioThreadPoolSize(4).build();
ClientResources rateLimitersCacheClientResources = ClientResources.builder().build();
FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", config.getCacheClusterConfiguration(), redisClientResources);
FaultTolerantRedisCluster messagesCluster = new FaultTolerantRedisCluster("messages_cluster", config.getMessageCacheConfiguration().getRedisClusterConfiguration(), redisClientResources);
FaultTolerantRedisCluster clientPresenceCluster = new FaultTolerantRedisCluster("client_presence_cluster", config.getClientPresenceClusterConfiguration(), redisClientResources);
FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", config.getMetricsClusterConfiguration(), redisClientResources);
FaultTolerantRedisCluster pushSchedulerCluster = new FaultTolerantRedisCluster("push_scheduler", config.getPushSchedulerCluster(), redisClientResources);
FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters", config.getRateLimitersCluster(), redisClientResources);
ConnectionEventLogger.logConnectionEvents(generalCacheClientResources);
ConnectionEventLogger.logConnectionEvents(messageCacheClientResources);
ConnectionEventLogger.logConnectionEvents(presenceClientResources);
ConnectionEventLogger.logConnectionEvents(metricsCacheClientResources);
FaultTolerantRedisCluster cacheCluster = new FaultTolerantRedisCluster("main_cache_cluster", config.getCacheClusterConfiguration(), generalCacheClientResources);
FaultTolerantRedisCluster messagesCluster = new FaultTolerantRedisCluster("messages_cluster", config.getMessageCacheConfiguration().getRedisClusterConfiguration(), messageCacheClientResources);
FaultTolerantRedisCluster clientPresenceCluster = new FaultTolerantRedisCluster("client_presence_cluster", config.getClientPresenceClusterConfiguration(), presenceClientResources);
FaultTolerantRedisCluster metricsCluster = new FaultTolerantRedisCluster("metrics_cluster", config.getMetricsClusterConfiguration(), metricsCacheClientResources);
FaultTolerantRedisCluster pushSchedulerCluster = new FaultTolerantRedisCluster("push_scheduler", config.getPushSchedulerCluster(), pushSchedulerCacheClientResources);
FaultTolerantRedisCluster rateLimitersCluster = new FaultTolerantRedisCluster("rate_limiters", config.getRateLimitersCluster(), rateLimitersCacheClientResources);
BlockingQueue<Runnable> keyspaceNotificationDispatchQueue = new ArrayBlockingQueue<>(10_000);
Metrics.gaugeCollectionSize(name(getClass(), "keyspaceNotificationDispatchQueueSize"), Collections.emptyList(), keyspaceNotificationDispatchQueue);
@@ -421,14 +421,11 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
build();
StripeManager stripeManager = new StripeManager(config.getStripe().getApiKey(), stripeExecutor,
config.getStripe().getIdempotencyKeyGenerator(), config.getStripe().getBoostDescription());
config.getStripe().getIdempotencyKeyGenerator());
ExternalServiceCredentialGenerator directoryCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenSharedSecret(),
config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenUserIdSecret());
ExternalServiceCredentialGenerator directoryV2CredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getDirectoryV2Configuration().getDirectoryV2ClientConfiguration()
.getUserAuthenticationTokenSharedSecret(), false);
ExternalServiceCredentialGenerator directoryCredentialsGenerator = new ExternalServiceCredentialGenerator(config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenSharedSecret(),
config.getDirectoryConfiguration().getDirectoryClientConfiguration().getUserAuthenticationTokenUserIdSecret(),
true);
DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager =
new DynamicConfigurationManager<>(config.getAppConfig().getApplication(),
@@ -443,12 +440,9 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
TwilioVerifyExperimentEnrollmentManager verifyExperimentEnrollmentManager = new TwilioVerifyExperimentEnrollmentManager(
config.getVoiceVerificationConfiguration(), experimentEnrollmentManager);
ExternalServiceCredentialGenerator storageCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getSecureStorageServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true);
ExternalServiceCredentialGenerator backupCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getSecureBackupServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true);
ExternalServiceCredentialGenerator paymentsCredentialsGenerator = new ExternalServiceCredentialGenerator(
config.getPaymentsServiceConfiguration().getUserAuthenticationTokenSharedSecret(), true);
ExternalServiceCredentialGenerator storageCredentialsGenerator = new ExternalServiceCredentialGenerator(config.getSecureStorageServiceConfiguration().getUserAuthenticationTokenSharedSecret(), new byte[0], false);
ExternalServiceCredentialGenerator backupCredentialsGenerator = new ExternalServiceCredentialGenerator(config.getSecureBackupServiceConfiguration().getUserAuthenticationTokenSharedSecret(), new byte[0], false);
ExternalServiceCredentialGenerator paymentsCredentialsGenerator = new ExternalServiceCredentialGenerator(config.getPaymentsServiceConfiguration().getUserAuthenticationTokenSharedSecret(), new byte[0], false);
SecureBackupClient secureBackupClient = new SecureBackupClient(backupCredentialsGenerator, backupServiceExecutor, config.getSecureBackupServiceConfiguration());
SecureStorageClient secureStorageClient = new SecureStorageClient(storageCredentialsGenerator, storageServiceExecutor, config.getSecureStorageServiceConfiguration());
@@ -456,24 +450,24 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
DirectoryQueue directoryQueue = new DirectoryQueue(config.getDirectoryConfiguration().getSqsConfiguration());
StoredVerificationCodeManager pendingAccountsManager = new StoredVerificationCodeManager(pendingAccounts);
StoredVerificationCodeManager pendingDevicesManager = new StoredVerificationCodeManager(pendingDevices);
UsernamesManager usernamesManager = new UsernamesManager(usernames, reservedUsernames, cacheCluster);
ProfilesManager profilesManager = new ProfilesManager(profiles, cacheCluster);
MessagesCache messagesCache = new MessagesCache(messagesCluster, messagesCluster, keyspaceNotificationDispatchExecutor);
PushLatencyManager pushLatencyManager = new PushLatencyManager(metricsCluster, dynamicConfigurationManager);
PushLatencyManager pushLatencyManager = new PushLatencyManager(metricsCluster);
ReportMessageManager reportMessageManager = new ReportMessageManager(reportMessageDynamoDb, rateLimitersCluster, Metrics.globalRegistry, config.getReportMessageConfiguration().getCounterTtl());
MessagesManager messagesManager = new MessagesManager(messagesDynamoDb, messagesCache, pushLatencyManager, reportMessageManager);
DeletedAccountsManager deletedAccountsManager = new DeletedAccountsManager(deletedAccounts,
deletedAccountsLockDynamoDbClient, config.getDeletedAccountsLockDynamoDbConfiguration().getTableName());
AccountsManager accountsManager = new AccountsManager(accounts, phoneNumberIdentifiers, cacheCluster,
deletedAccountsManager, directoryQueue, keys, messagesManager, reservedUsernames, profilesManager,
pendingAccountsManager, secureStorageClient, secureBackupClient, clientPresenceManager, clock);
RemoteConfigsManager remoteConfigsManager = new RemoteConfigsManager(remoteConfigs);
AccountsManager accountsManager = new AccountsManager(accounts, cacheCluster,
deletedAccountsManager, directoryQueue, keysDynamoDb, messagesManager, usernamesManager, profilesManager,
pendingAccountsManager, secureStorageClient, secureBackupClient, clientPresenceManager);
RemoteConfigsManager remoteConfigsManager = new RemoteConfigsManager(remoteConfigs);
DeadLetterHandler deadLetterHandler = new DeadLetterHandler(accountsManager, messagesManager);
DispatchManager dispatchManager = new DispatchManager(pubSubClientFactory, Optional.of(deadLetterHandler));
PubSubManager pubSubManager = new PubSubManager(pubsubClient, dispatchManager);
APNSender apnSender = new APNSender(apnSenderExecutor, accountsManager, config.getApnConfiguration());
GCMSender gcmSender = new GCMSender(gcmSenderExecutor, accountsManager, config.getGcmConfiguration().getApiKey());
RateLimiters rateLimiters = new RateLimiters(config.getLimitsConfiguration(), rateLimitersCluster);
DynamicRateLimiters dynamicRateLimiters = new DynamicRateLimiters(rateLimitersCluster, dynamicConfigurationManager);
RateLimiters rateLimiters = new RateLimiters(config.getLimitsConfiguration(), dynamicConfigurationManager, rateLimitersCluster);
ProvisioningManager provisioningManager = new ProvisioningManager(pubSubManager);
IssuedReceiptsManager issuedReceiptsManager = new IssuedReceiptsManager(
config.getDynamoDbTables().getIssuedReceipts().getTableName(),
@@ -491,6 +485,11 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
AccountAuthenticator accountAuthenticator = new AccountAuthenticator(accountsManager);
DisabledPermittedAccountAuthenticator disabledPermittedAccountAuthenticator = new DisabledPermittedAccountAuthenticator(accountsManager);
RateLimitResetMetricsManager rateLimitResetMetricsManager = new RateLimitResetMetricsManager(metricsCluster, Metrics.globalRegistry);
UnsealedSenderRateLimiter unsealedSenderRateLimiter = new UnsealedSenderRateLimiter(rateLimiters, rateLimitersCluster, dynamicConfigurationManager, rateLimitResetMetricsManager);
PreKeyRateLimiter preKeyRateLimiter = new PreKeyRateLimiter(rateLimiters, dynamicConfigurationManager, rateLimitResetMetricsManager);
ApnFallbackManager apnFallbackManager = new ApnFallbackManager(pushSchedulerCluster, apnSender, accountsManager);
TwilioSmsSender twilioSmsSender = new TwilioSmsSender(config.getTwilioConfiguration(), dynamicConfigurationManager);
SmsSender smsSender = new SmsSender(twilioSmsSender);
@@ -506,13 +505,14 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
TransitionalRecaptchaClient transitionalRecaptchaClient = new TransitionalRecaptchaClient(legacyRecaptchaClient, enterpriseRecaptchaClient);
PushChallengeManager pushChallengeManager = new PushChallengeManager(apnSender, gcmSender, pushChallengeDynamoDb);
RateLimitChallengeManager rateLimitChallengeManager = new RateLimitChallengeManager(pushChallengeManager,
transitionalRecaptchaClient, dynamicRateLimiters);
RateLimitChallengeOptionManager rateLimitChallengeOptionManager =
new RateLimitChallengeOptionManager(dynamicRateLimiters, dynamicConfigurationManager);
transitionalRecaptchaClient, preKeyRateLimiter, unsealedSenderRateLimiter, rateLimiters,
dynamicConfigurationManager);
MessagePersister messagePersister = new MessagePersister(messagesCache, messagesManager, accountsManager, dynamicConfigurationManager, Duration.ofMinutes(config.getMessageCacheConfiguration().getPersistDelayMinutes()));
final List<AccountDatabaseCrawlerListener> directoryReconciliationAccountDatabaseCrawlerListeners = new ArrayList<>();
// TODO listeners must be ordered so that ones that directly update accounts come last, so that read-only ones are not working with stale data
final List<AccountDatabaseCrawlerListener> accountDatabaseCrawlerListeners = new ArrayList<>();
final List<DeletedAccountsDirectoryReconciler> deletedAccountsDirectoryReconcilers = new ArrayList<>();
for (DirectoryServerConfiguration directoryServerConfiguration : config.getDirectoryConfiguration()
.getDirectoryServerConfiguration()) {
@@ -522,37 +522,26 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
directoryServerConfiguration.getReplicationName(), directoryReconciliationClient,
dynamicConfigurationManager);
// reconcilers are read-only
directoryReconciliationAccountDatabaseCrawlerListeners.add(directoryReconciler);
accountDatabaseCrawlerListeners.add(directoryReconciler);
final DeletedAccountsDirectoryReconciler deletedAccountsDirectoryReconciler = new DeletedAccountsDirectoryReconciler(
directoryServerConfiguration.getReplicationName(), directoryReconciliationClient);
deletedAccountsDirectoryReconcilers.add(deletedAccountsDirectoryReconciler);
}
accountDatabaseCrawlerListeners.add(new NonNormalizedAccountCrawlerListener(accountsManager, metricsCluster));
accountDatabaseCrawlerListeners.add(new ContactDiscoveryWriter(accountsManager));
// PushFeedbackProcessor may update device properties
accountDatabaseCrawlerListeners.add(new PushFeedbackProcessor(accountsManager));
// delete accounts last
accountDatabaseCrawlerListeners.add(new AccountCleaner(accountsManager));
AccountDatabaseCrawlerCache directoryReconciliationAccountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
cacheCluster, AccountDatabaseCrawlerCache.DIRECTORY_RECONCILER_PREFIX);
AccountDatabaseCrawler directoryReconciliationAccountDatabaseCrawler = new AccountDatabaseCrawler(
"Reconciliation crawler",
accountsManager,
directoryReconciliationAccountDatabaseCrawlerCache, directoryReconciliationAccountDatabaseCrawlerListeners,
config.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
config.getAccountDatabaseCrawlerConfiguration().getChunkIntervalMs()
);
HttpClient currencyClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).connectTimeout(Duration.ofSeconds(10)).build();
FixerClient fixerClient = new FixerClient(currencyClient, config.getPaymentsServiceConfiguration().getFixerApiKey());
FtxClient ftxClient = new FtxClient(currencyClient);
CurrencyConversionManager currencyManager = new CurrencyConversionManager(fixerClient, ftxClient, config.getPaymentsServiceConfiguration().getPaymentCurrencies());
// TODO listeners must be ordered so that ones that directly update accounts come last, so that read-only ones are not working with stale data
final List<AccountDatabaseCrawlerListener> accountDatabaseCrawlerListeners = List.of(
new NonNormalizedAccountCrawlerListener(accountsManager, metricsCluster),
new ContactDiscoveryWriter(accountsManager),
// PushFeedbackProcessor may update device properties
new PushFeedbackProcessor(accountsManager),
// delete accounts last
new AccountCleaner(accountsManager)
);
AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(cacheCluster,
AccountDatabaseCrawlerCache.GENERAL_PURPOSE_PREFIX);
AccountDatabaseCrawler accountDatabaseCrawler = new AccountDatabaseCrawler("General-purpose account crawler",
accountsManager,
AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(cacheCluster);
AccountDatabaseCrawler accountDatabaseCrawler = new AccountDatabaseCrawler(accountsManager,
accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners,
config.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
config.getAccountDatabaseCrawlerConfiguration().getChunkIntervalMs()
@@ -560,19 +549,14 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
DeletedAccountsTableCrawler deletedAccountsTableCrawler = new DeletedAccountsTableCrawler(deletedAccountsManager, deletedAccountsDirectoryReconcilers, cacheCluster, recurringJobExecutor);
HttpClient currencyClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).connectTimeout(Duration.ofSeconds(10)).build();
FixerClient fixerClient = new FixerClient(currencyClient, config.getPaymentsServiceConfiguration().getFixerApiKey());
FtxClient ftxClient = new FtxClient(currencyClient);
CurrencyConversionManager currencyManager = new CurrencyConversionManager(fixerClient, ftxClient, config.getPaymentsServiceConfiguration().getPaymentCurrencies());
apnSender.setApnFallbackManager(apnFallbackManager);
environment.lifecycle().manage(new ApplicationShutdownMonitor());
environment.lifecycle().manage(apnFallbackManager);
environment.lifecycle().manage(pubSubManager);
environment.lifecycle().manage(messageSender);
environment.lifecycle().manage(accountDatabaseCrawler);
environment.lifecycle().manage(directoryReconciliationAccountDatabaseCrawler);
environment.lifecycle().manage(deletedAccountsTableCrawler);
environment.lifecycle().manage(remoteConfigsManager);
environment.lifecycle().manage(messagesCache);
environment.lifecycle().manage(messagePersister);
environment.lifecycle().manage(clientPresenceManager);
@@ -606,12 +590,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
.addFilter("RemoteDeprecationFilter", new RemoteDeprecationFilter(dynamicConfigurationManager))
.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/*");
// TODO Remove on or after 2022-03-01
final AcceptNumericOnlineFlagRequestFilter acceptNumericOnlineFlagRequestFilter =
new AcceptNumericOnlineFlagRequestFilter("v1/messages/multi_recipient");
environment.jersey().register(new ContentLengthFilter(TrafficSource.HTTP));
environment.jersey().register(acceptNumericOnlineFlagRequestFilter);
environment.jersey().register(MultiRecipientMessageProvider.class);
environment.jersey().register(new MetricsApplicationEventListener(TrafficSource.HTTP));
environment.jersey()
@@ -633,18 +612,17 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
clientPresenceManager, retrySchedulingExecutor));
webSocketEnvironment.jersey().register(new WebsocketRefreshApplicationEventListener(accountsManager, clientPresenceManager));
webSocketEnvironment.jersey().register(new ContentLengthFilter(TrafficSource.WEBSOCKET));
webSocketEnvironment.jersey().register(acceptNumericOnlineFlagRequestFilter);
webSocketEnvironment.jersey().register(MultiRecipientMessageProvider.class);
webSocketEnvironment.jersey().register(new MetricsApplicationEventListener(TrafficSource.WEBSOCKET));
webSocketEnvironment.jersey().register(new KeepAliveController(clientPresenceManager));
// these should be common, but use @Auth DisabledPermittedAccount, which isn’t supported yet on websocket
environment.jersey().register(
new AccountController(pendingAccountsManager, accountsManager, abusiveHostRules, rateLimiters,
new AccountController(pendingAccountsManager, accountsManager, usernamesManager, abusiveHostRules, rateLimiters,
smsSender, dynamicConfigurationManager, turnTokenGenerator, config.getTestDevices(),
transitionalRecaptchaClient, gcmSender, apnSender, backupCredentialsGenerator,
verifyExperimentEnrollmentManager));
environment.jersey().register(new KeysController(rateLimiters, keys, accountsManager));
environment.jersey().register(new KeysController(rateLimiters, keysDynamoDb, accountsManager, preKeyRateLimiter, rateLimitChallengeManager));
final List<Object> commonControllers = Lists.newArrayList(
new AttachmentControllerV1(rateLimiters, config.getAwsAttachmentsConfiguration().getAccessKey(), config.getAwsAttachmentsConfiguration().getAccessSecret(), config.getAwsAttachmentsConfiguration().getBucket()),
@@ -652,15 +630,14 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
new AttachmentControllerV3(rateLimiters, config.getGcpAttachmentsConfiguration().getDomain(), config.getGcpAttachmentsConfiguration().getEmail(), config.getGcpAttachmentsConfiguration().getMaxSizeInBytes(), config.getGcpAttachmentsConfiguration().getPathPrefix(), config.getGcpAttachmentsConfiguration().getRsaSigningKey()),
new CertificateController(new CertificateGenerator(config.getDeliveryCertificate().getCertificate(), config.getDeliveryCertificate().getPrivateKey(), config.getDeliveryCertificate().getExpiresDays()), zkAuthOperations),
new ChallengeController(rateLimitChallengeManager),
new DeviceController(pendingDevicesManager, accountsManager, messagesManager, keys, rateLimiters, config.getMaxDevices()),
new DeviceController(pendingDevicesManager, accountsManager, messagesManager, keysDynamoDb, rateLimiters, config.getMaxDevices()),
new DirectoryController(directoryCredentialsGenerator),
new DirectoryV2Controller(directoryV2CredentialsGenerator),
new DonationController(clock, zkReceiptOperations, redeemedReceiptsManager, accountsManager, config.getBadges(),
ReceiptCredentialPresentation::new, stripeExecutor, config.getDonationConfiguration(), config.getStripe()),
new MessageController(rateLimiters, messageSender, receiptSender, accountsManager, messagesManager, apnFallbackManager,
reportMessageManager, multiRecipientMessageExecutor),
new MessageController(rateLimiters, messageSender, receiptSender, accountsManager, messagesManager, unsealedSenderRateLimiter, apnFallbackManager,
rateLimitChallengeManager, reportMessageManager, multiRecipientMessageExecutor),
new PaymentsController(currencyManager, paymentsCredentialsGenerator),
new ProfileController(clock, rateLimiters, accountsManager, profilesManager, dynamicConfigurationManager, profileBadgeConverter, config.getBadges(), cdnS3Client, profileCdnPolicyGenerator, profileCdnPolicySigner, config.getCdnConfiguration().getBucket(), zkProfileOperations),
new ProfileController(clock, rateLimiters, accountsManager, profilesManager, usernamesManager, dynamicConfigurationManager, profileBadgeConverter, config.getBadges(), cdnS3Client, profileCdnPolicyGenerator, profileCdnPolicySigner, config.getCdnConfiguration().getBucket(), zkProfileOperations),
new ProvisioningController(rateLimiters, provisioningManager),
new RemoteConfigController(remoteConfigsManager, config.getRemoteConfigConfiguration().getAuthorizedTokens(), config.getRemoteConfigConfiguration().getGlobalConfig()),
new SecureBackupController(backupCredentialsGenerator),
@@ -700,11 +677,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
log.warn("Abusive message filter {} not annotated with @FilterAbusiveMessages and will not be installed",
filter.getClass().getName());
}
if (filter instanceof RateLimitChallengeListener) {
log.info("Registered rate limit challenge listener: {}", filter.getClass().getName());
rateLimitChallengeManager.addListener((RateLimitChallengeListener) filter);
}
}
if (!registeredAbusiveMessageFilter) {
@@ -721,8 +693,8 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
registerCorsFilter(environment);
registerExceptionMappers(environment, webSocketEnvironment, provisioningEnvironment);
RateLimitChallengeExceptionMapper rateLimitChallengeExceptionMapper =
new RateLimitChallengeExceptionMapper(rateLimitChallengeOptionManager);
RateLimitChallengeExceptionMapper rateLimitChallengeExceptionMapper = new RateLimitChallengeExceptionMapper(
rateLimitChallengeManager);
environment.jersey().register(rateLimitChallengeExceptionMapper);
webSocketEnvironment.jersey().register(rateLimitChallengeExceptionMapper);
@@ -772,7 +744,6 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
List.of(
new LoggingUnhandledExceptionMapper(),
new CompletionExceptionMapper(),
new IOExceptionMapper(),
new RateLimitExceededExceptionMapper(),
new InvalidWebsocketAddressExceptionMapper(),
@@ -1,24 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.abuse;
import org.whispersystems.textsecuregcm.storage.Account;
import java.io.IOException;
public interface RateLimitChallengeListener {
void handleRateLimitChallengeAnswered(Account account);
/**
* Configures this rate limit challenge listener. This method will be called before the service begins processing any
* challenges.
*
* @param environmentName the name of the environment in which this listener is running (e.g. "staging" or "production")
* @throws IOException if the listener could not read its configuration source for any reason
*/
void configure(String environmentName) throws IOException;
}
@@ -1,12 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.abuse;
public enum RateLimitChallengeType {
PUSH_CHALLENGE,
RECAPTCHA
}
@@ -76,7 +76,7 @@ public class AuthEnablementRefreshRequirementProvider implements WebsocketRefres
@SuppressWarnings("unchecked") final Map<Long, Boolean> initialDevicesEnabled =
(Map<Long, Boolean>) requestEvent.getContainerRequest().getProperty(DEVICES_ENABLED);
return accountsManager.getByAccountIdentifier((UUID) requestEvent.getContainerRequest().getProperty(ACCOUNT_UUID)).map(account -> {
return accountsManager.get((UUID) requestEvent.getContainerRequest().getProperty(ACCOUNT_UUID)).map(account -> {
final Set<Long> deviceIdsToDisplace;
final Map<Long, Boolean> currentDevicesEnabled = buildDevicesEnabledMap(account);
@@ -78,7 +78,7 @@ public class BaseAccountAuthenticator {
deviceId = identifierAndDeviceId.second();
}
Optional<Account> account = accountsManager.getByAccountIdentifier(accountUuid);
Optional<Account> account = accountsManager.get(accountUuid);
if (account.isEmpty()) {
failureReason = "noSuchAccount";
@@ -1,71 +1,97 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.auth;
import com.google.common.annotations.VisibleForTesting;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.time.Clock;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.Util;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.security.InvalidKeyException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.TimeUnit;
public class ExternalServiceCredentialGenerator {
private final Logger logger = LoggerFactory.getLogger(ExternalServiceCredentialGenerator.class);
private final byte[] key;
private final byte[] userIdKey;
private final boolean usernameDerivation;
private final boolean prependUsername;
private final Clock clock;
public ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey) {
this(key, userIdKey, true, true);
}
public ExternalServiceCredentialGenerator(byte[] key, boolean prependUsername) {
this(key, new byte[0], false, prependUsername);
}
@VisibleForTesting
public ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey, boolean usernameDerivation) {
this(key, userIdKey, usernameDerivation, true);
}
private ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey, boolean usernameDerivation,
boolean prependUsername) {
this(key, userIdKey, usernameDerivation, prependUsername, Clock.systemUTC());
}
@VisibleForTesting
public ExternalServiceCredentialGenerator(byte[] key, byte[] userIdKey, boolean usernameDerivation,
boolean prependUsername, Clock clock) {
this.key = key;
this.userIdKey = userIdKey;
this.key = key;
this.userIdKey = userIdKey;
this.usernameDerivation = usernameDerivation;
this.prependUsername = prependUsername;
this.clock = clock;
}
public ExternalServiceCredentials generateFor(String identity) {
Mac mac = getMacInstance();
String username = getUserId(identity, mac, usernameDerivation);
long currentTimeSeconds = clock.millis() / 1000;
String prefix = username + ":" + currentTimeSeconds;
String output = Hex.encodeHexString(Util.truncate(getHmac(key, prefix.getBytes(), mac), 10));
String token = (prependUsername ? prefix : currentTimeSeconds) + ":" + output;
public ExternalServiceCredentials generateFor(String number) {
Mac mac = getMacInstance();
String username = getUserId(number, mac, usernameDerivation);
long currentTimeSeconds = System.currentTimeMillis() / 1000;
String prefix = username + ":" + currentTimeSeconds;
String output = Hex.encodeHexString(Util.truncate(getHmac(key, prefix.getBytes(), mac), 10));
String token = prefix + ":" + output;
return new ExternalServiceCredentials(username, token);
}
public boolean isValid(String token, String number, long currentTimeMillis) {
String[] parts = token.split(":");
Mac mac = getMacInstance();
if (parts.length != 3) {
return false;
}
if (!getUserId(number, mac, usernameDerivation).equals(parts[0])) {
return false;
}
if (!isValidTime(parts[1], currentTimeMillis)) {
return false;
}
return isValidSignature(parts[0] + ":" + parts[1], parts[2], mac);
}
private String getUserId(String number, Mac mac, boolean usernameDerivation) {
if (usernameDerivation) return Hex.encodeHexString(Util.truncate(getHmac(userIdKey, number.getBytes(), mac), 10));
else return number;
}
private boolean isValidTime(String timeString, long currentTimeMillis) {
try {
long tokenTime = Long.parseLong(timeString);
long ourTime = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis);
return TimeUnit.SECONDS.toHours(Math.abs(ourTime - tokenTime)) < 24;
} catch (NumberFormatException e) {
logger.warn("Number Format", e);
return false;
}
}
private boolean isValidSignature(String prefix, String suffix, Mac mac) {
try {
byte[] ourSuffix = Util.truncate(getHmac(key, prefix.getBytes(), mac), 10);
byte[] theirSuffix = Hex.decodeHex(suffix.toCharArray());
return MessageDigest.isEqual(ourSuffix, theirSuffix);
} catch (DecoderException e) {
logger.warn("DirectoryCredentials", e);
return false;
}
}
private Mac getMacInstance() {
try {
return Mac.getInstance("HmacSHA256");
@@ -5,14 +5,10 @@
package org.whispersystems.textsecuregcm.auth;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicInteger;
import javax.ws.rs.container.ResourceInfo;
import javax.ws.rs.core.Context;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Metrics;
import org.glassfish.jersey.server.monitoring.RequestEvent;
import org.glassfish.jersey.server.monitoring.RequestEvent.Type;
import org.glassfish.jersey.server.monitoring.RequestEventListener;
@@ -20,6 +16,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.push.ClientPresenceManager;
import javax.ws.rs.container.ResourceInfo;
import javax.ws.rs.core.Context;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
public class WebsocketRefreshRequestEventListener implements RequestEventListener {
private final ClientPresenceManager clientPresenceManager;
@@ -59,7 +60,7 @@ public class WebsocketRefreshRequestEventListener implements RequestEventListene
.forEach(pair -> {
try {
displacedDevices.incrementAndGet();
clientPresenceManager.disconnectPresence(pair.first(), pair.second());
clientPresenceManager.displacePresence(pair.first(), pair.second());
} catch (final Exception e) {
logger.error("Could not displace device presence", e);
}
@@ -0,0 +1,23 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
public class AccountsDatabaseConfiguration extends DatabaseConfiguration {
@JsonProperty
@NotNull
@Valid
private RetryConfiguration keyOperationRetry = new RetryConfiguration();
public RetryConfiguration getKeyOperationRetryConfiguration() {
return keyOperationRetry;
}
}
@@ -8,12 +8,6 @@ public class AccountsDynamoDbConfiguration extends DynamoDbConfiguration {
@NotNull
private String phoneNumberTableName;
@NotNull
private String phoneNumberIdentifierTableName;
@NotNull
private String usernamesTableName;
private int scanPageSize = 100;
@JsonProperty
@@ -21,18 +15,9 @@ public class AccountsDynamoDbConfiguration extends DynamoDbConfiguration {
return phoneNumberTableName;
}
@JsonProperty
public String getPhoneNumberIdentifierTableName() {
return phoneNumberIdentifierTableName;
}
@JsonProperty
public String getUsernamesTableName() {
return usernamesTableName;
}
@JsonProperty
public int getScanPageSize() {
return scanPageSize;
}
}
@@ -1,26 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.whispersystems.textsecuregcm.util.ExactlySize;
public class DirectoryV2ClientConfiguration {
private final byte[] userAuthenticationTokenSharedSecret;
@JsonCreator
public DirectoryV2ClientConfiguration(
@JsonProperty("userAuthenticationTokenSharedSecret") final byte[] userAuthenticationTokenSharedSecret) {
this.userAuthenticationTokenSharedSecret = userAuthenticationTokenSharedSecret;
}
@ExactlySize({32})
public byte[] getUserAuthenticationTokenSharedSecret() {
return userAuthenticationTokenSharedSecret;
}
}
@@ -1,24 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import javax.validation.Valid;
public class DirectoryV2Configuration {
private final DirectoryV2ClientConfiguration clientConfiguration;
@JsonCreator
public DirectoryV2Configuration(@JsonProperty("client") DirectoryV2ClientConfiguration clientConfiguration) {
this.clientConfiguration = clientConfiguration;
}
@Valid
public DirectoryV2ClientConfiguration getDirectoryV2ClientConfiguration() {
return clientConfiguration;
}
}
@@ -49,21 +49,15 @@ public class DynamoDbTables {
private final IssuedReceiptsTableConfiguration issuedReceipts;
private final TableWithExpiration redeemedReceipts;
private final Table subscriptions;
private final Table profiles;
private final Table remoteConfig;
@JsonCreator
public DynamoDbTables(
@JsonProperty("issuedReceipts") final IssuedReceiptsTableConfiguration issuedReceipts,
@JsonProperty("redeemedReceipts") final TableWithExpiration redeemedReceipts,
@JsonProperty("subscriptions") final Table subscriptions,
@JsonProperty("profiles") final Table profiles,
@JsonProperty("remoteConfig") final Table remoteConfig) {
@JsonProperty("subscriptions") final Table subscriptions) {
this.issuedReceipts = issuedReceipts;
this.redeemedReceipts = redeemedReceipts;
this.subscriptions = subscriptions;
this.profiles = profiles;
this.remoteConfig = remoteConfig;
}
@Valid
@@ -83,16 +77,4 @@ public class DynamoDbTables {
public Table getSubscriptions() {
return subscriptions;
}
@Valid
@NotNull
public Table getProfiles() {
return profiles;
}
@Valid
@NotNull
public Table getRemoteConfig() {
return remoteConfig;
}
}
@@ -63,9 +63,6 @@ public class RateLimitsConfiguration {
@JsonProperty
private RateLimitConfiguration usernameSet = new RateLimitConfiguration(100, 100 / (24.0 * 60.0));
@JsonProperty
private RateLimitConfiguration checkAccountExistence = new RateLimitConfiguration(1_000, 1_000 / 60.0);
public RateLimitConfiguration getAutoBlock() {
return autoBlock;
}
@@ -138,10 +135,6 @@ public class RateLimitsConfiguration {
return usernameSet;
}
public RateLimitConfiguration getCheckAccountExistence() {
return checkAccountExistence;
}
public static class RateLimitConfiguration {
@JsonProperty
private int bucketSize;
@@ -13,16 +13,13 @@ public class StripeConfiguration {
private final String apiKey;
private final byte[] idempotencyKeyGenerator;
private final String boostDescription;
@JsonCreator
public StripeConfiguration(
@JsonProperty("apiKey") final String apiKey,
@JsonProperty("idempotencyKeyGenerator") final byte[] idempotencyKeyGenerator,
@JsonProperty("boostDescription") final String boostDescription) {
@JsonProperty("idempotencyKeyGenerator") final byte[] idempotencyKeyGenerator) {
this.apiKey = apiKey;
this.idempotencyKeyGenerator = idempotencyKeyGenerator;
this.boostDescription = boostDescription;
}
@NotEmpty
@@ -34,9 +31,4 @@ public class StripeConfiguration {
public byte[] getIdempotencyKeyGenerator() {
return idempotencyKeyGenerator;
}
@NotEmpty
public String getBoostDescription() {
return boostDescription;
}
}
@@ -51,14 +51,6 @@ public class DynamicConfiguration {
@JsonProperty
private DynamicDirectoryReconcilerConfiguration directoryReconciler = new DynamicDirectoryReconcilerConfiguration();
@JsonProperty
@Valid
private DynamicPushLatencyConfiguration pushLatency = new DynamicPushLatencyConfiguration(Collections.emptyMap());
@JsonProperty
@Valid
private DynamicProfileMigrationConfiguration profileMigration = new DynamicProfileMigrationConfiguration();
public Optional<DynamicExperimentEnrollmentConfiguration> getExperimentEnrollmentConfiguration(
final String experimentName) {
return Optional.ofNullable(experiments.get(experimentName));
@@ -109,12 +101,4 @@ public class DynamicConfiguration {
public DynamicDirectoryReconcilerConfiguration getDirectoryReconcilerConfiguration() {
return directoryReconciler;
}
public DynamicPushLatencyConfiguration getPushLatencyConfiguration() {
return pushLatency;
}
public DynamicProfileMigrationConfiguration getProfileMigrationConfiguration() {
return profileMigration;
}
}
@@ -7,14 +7,14 @@ package org.whispersystems.textsecuregcm.configuration.dynamic;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Collections;
import java.util.List;
import java.util.Set;
public class DynamicPaymentsConfiguration {
@JsonProperty
private List<String> disallowedPrefixes = Collections.emptyList();
private Set<String> allowedCountryCodes = Collections.emptySet();
public List<String> getDisallowedPrefixes() {
return disallowedPrefixes;
public Set<String> getAllowedCountryCodes() {
return allowedCountryCodes;
}
}
@@ -1,46 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration.dynamic;
import com.fasterxml.jackson.annotation.JsonProperty;
public class DynamicProfileMigrationConfiguration {
@JsonProperty
private boolean dynamoDbDeleteEnabled = false;
@JsonProperty
private boolean dynamoDbWriteEnabled = false;
@JsonProperty
private boolean dynamoDbReadForComparisonEnabled = false;
@JsonProperty
private boolean dynamoDbReadPrimary = false;
@JsonProperty
private boolean logMismatches = false;
public boolean isDynamoDbDeleteEnabled() {
return dynamoDbDeleteEnabled;
}
public boolean isDynamoDbWriteEnabled() {
return dynamoDbWriteEnabled;
}
public boolean isDynamoDbReadForComparisonEnabled() {
return dynamoDbReadForComparisonEnabled;
}
public boolean isDynamoDbReadPrimary() {
return dynamoDbReadPrimary;
}
public boolean isLogMismatches() {
return logMismatches;
}
}
@@ -1,27 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.configuration.dynamic;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.vdurmont.semver4j.Semver;
import org.whispersystems.textsecuregcm.util.ua.ClientPlatform;
import java.util.Map;
import java.util.Set;
public class DynamicPushLatencyConfiguration {
private final Map<ClientPlatform, Set<Semver>> instrumentedVersions;
@JsonCreator
public DynamicPushLatencyConfiguration(@JsonProperty("instrumentedVersions") final Map<ClientPlatform, Set<Semver>> instrumentedVersions) {
this.instrumentedVersions = instrumentedVersions;
}
public Map<ClientPlatform, Set<Semver>> getInstrumentedVersions() {
return instrumentedVersions;
}
}
@@ -11,6 +11,9 @@ import javax.validation.constraints.NotNull;
public class DynamicRateLimitChallengeConfiguration {
@JsonProperty
private boolean preKeyLimitEnforced = false;
@JsonProperty
boolean unsealedSenderLimitEnforced = false;
@@ -27,6 +30,10 @@ public class DynamicRateLimitChallengeConfiguration {
return Optional.ofNullable(clientSupportedVersions.get(platform));
}
public boolean isPreKeyLimitEnforced() {
return preKeyLimitEnforced;
}
public boolean isUnsealedSenderLimitEnforced() {
return unsealedSenderLimitEnforced;
}
@@ -16,6 +16,9 @@ public class DynamicRateLimitsConfiguration {
@JsonProperty
private int unsealedSenderPermitIncrement = 50;
@JsonProperty
private RateLimitConfiguration unsealedSenderIp = new RateLimitConfiguration(120, 2.0 / 60);
@JsonProperty
private RateLimitConfiguration rateLimitReset = new RateLimitConfiguration(2, 2.0 / (60 * 24));
@@ -31,6 +34,13 @@ public class DynamicRateLimitsConfiguration {
@JsonProperty
private RateLimitConfiguration pushChallengeSuccess = new RateLimitConfiguration(2, 2.0 / (60 * 24));
@JsonProperty
private RateLimitConfiguration dailyPreKeys = new RateLimitConfiguration(50, 50.0 / (24.0 * 60));
public RateLimitConfiguration getUnsealedSenderIp() {
return unsealedSenderIp;
}
public CardinalityRateLimitConfiguration getUnsealedSenderNumber() {
return unsealedSenderNumber;
}
@@ -62,4 +72,8 @@ public class DynamicRateLimitsConfiguration {
public int getUnsealedSenderPermitIncrement() {
return unsealedSenderPermitIncrement;
}
public RateLimitConfiguration getDailyPreKeys() {
return dailyPreKeys;
}
}
@@ -1,32 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.controllers;
import java.io.IOException;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import javax.ws.rs.container.PreMatching;
@PreMatching
public class AcceptNumericOnlineFlagRequestFilter implements ContainerRequestFilter {
private final String pathPrefix;
public AcceptNumericOnlineFlagRequestFilter(final String pathPrefix) {
this.pathPrefix = pathPrefix;
}
@Override
public void filter(final ContainerRequestContext requestContext) throws IOException {
if (requestContext.getUriInfo().getPath().startsWith(pathPrefix)) {
if ("1".equals(requestContext.getUriInfo().getQueryParameters().getFirst("online"))) {
requestContext.setRequestUri(requestContext.getUriInfo().getRequestUriBuilder()
.replaceQueryParam("online", "true")
.build());
}
}
}
}
@@ -15,7 +15,6 @@ import io.dropwizard.auth.Auth;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import java.security.SecureRandom;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
@@ -23,17 +22,13 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import javax.servlet.http.HttpServletRequest;
import javax.validation.Valid;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.HEAD;
import javax.ws.rs.HeaderParam;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
@@ -41,11 +36,8 @@ import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
@@ -85,14 +77,12 @@ import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
import org.whispersystems.textsecuregcm.storage.UsernameNotAvailableException;
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.ForwardedIpUtil;
import org.whispersystems.textsecuregcm.util.Hex;
import org.whispersystems.textsecuregcm.util.ImpossiblePhoneNumberException;
import org.whispersystems.textsecuregcm.util.NonNormalizedPhoneNumberException;
import org.whispersystems.textsecuregcm.util.Username;
import org.whispersystems.textsecuregcm.util.UsernameValidator;
import org.whispersystems.textsecuregcm.util.Util;
import org.whispersystems.textsecuregcm.util.VerificationCode;
@@ -125,6 +115,7 @@ public class AccountController {
private final StoredVerificationCodeManager pendingAccounts;
private final AccountsManager accounts;
private final UsernamesManager usernames;
private final AbusiveHostRules abusiveHostRules;
private final RateLimiters rateLimiters;
private final SmsSender smsSender;
@@ -140,6 +131,7 @@ public class AccountController {
public AccountController(StoredVerificationCodeManager pendingAccounts,
AccountsManager accounts,
UsernamesManager usernames,
AbusiveHostRules abusiveHostRules,
RateLimiters rateLimiters,
SmsSender smsSenderFactory,
@@ -154,6 +146,7 @@ public class AccountController {
{
this.pendingAccounts = pendingAccounts;
this.accounts = accounts;
this.usernames = usernames;
this.abusiveHostRules = abusiveHostRules;
this.rateLimiters = rateLimiters;
this.smsSender = smsSenderFactory;
@@ -360,11 +353,11 @@ public class AccountController {
storedVerificationCode.flatMap(StoredVerificationCode::getTwilioVerificationSid)
.ifPresent(smsSender::reportVerificationSucceeded);
Optional<Account> existingAccount = accounts.getByE164(number);
Optional<Account> existingAccount = accounts.get(number);
if (existingAccount.isPresent()) {
verifyRegistrationLock(existingAccount.get(), accountAttributes.getRegistrationLock());
}
if (existingAccount.isPresent()) {
verifyRegistrationLock(existingAccount.get(), accountAttributes.getRegistrationLock());
}
if (availableForTransfer.orElse(false) && existingAccount.map(Account::isTransferSupported).orElse(false)) {
throw new WebApplicationException(Response.status(409).build());
@@ -372,8 +365,7 @@ public class AccountController {
rateLimiters.getVerifyLimiter().clear(number);
Account account = accounts.create(number, password, signalAgent, accountAttributes,
existingAccount.map(Account::getBadges).orElseGet(ArrayList::new));
Account account = accounts.create(number, password, signalAgent, accountAttributes);
{
metricRegistry.meter(name(AccountController.class, "verify", Util.getCountryCode(number))).mark();
@@ -389,10 +381,7 @@ public class AccountController {
.record(Instant.now().toEpochMilli() - storedVerificationCode.get().getTimestamp(), TimeUnit.MILLISECONDS);
}
return new AccountCreationResult(account.getUuid(),
account.getNumber(),
account.getPhoneNumberIdentifier(),
existingAccount.map(Account::isStorageSupported).orElse(false));
return new AccountCreationResult(account.getUuid(), account.getNumber(), existingAccount.map(Account::isStorageSupported).orElse(false));
}
@Timed
@@ -422,7 +411,7 @@ public class AccountController {
storedVerificationCode.flatMap(StoredVerificationCode::getTwilioVerificationSid)
.ifPresent(smsSender::reportVerificationSucceeded);
final Optional<Account> existingAccount = accounts.getByE164(request.getNumber());
final Optional<Account> existingAccount = accounts.get(request.getNumber());
if (existingAccount.isPresent()) {
verifyRegistrationLock(existingAccount.get(), request.getRegistrationLock());
@@ -554,7 +543,6 @@ public class AccountController {
@PUT
@Path("/attributes/")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ChangesDeviceEnabledState
public void setAccountAttributes(@Auth DisabledPermittedAuthenticatedAccount disabledPermittedAuth,
@HeaderParam("X-Signal-Agent") String userAgent,
@@ -603,59 +591,40 @@ public class AccountController {
@Path("/whoami")
@Produces(MediaType.APPLICATION_JSON)
public AccountCreationResult whoAmI(@Auth AuthenticatedAccount auth) {
return new AccountCreationResult(auth.getAccount().getUuid(),
auth.getAccount().getNumber(),
auth.getAccount().getPhoneNumberIdentifier(),
auth.getAccount().isStorageSupported());
return new AccountCreationResult(auth.getAccount().getUuid(), auth.getAccount().getNumber(), auth.getAccount().isStorageSupported());
}
@DELETE
@Path("/username")
@Produces(MediaType.APPLICATION_JSON)
public void deleteUsername(@Auth AuthenticatedAccount auth) {
accounts.clearUsername(auth.getAccount());
usernames.delete(auth.getAccount().getUuid());
}
@PUT
@Path("/username/{username}")
@Produces(MediaType.APPLICATION_JSON)
public Response setUsername(@Auth AuthenticatedAccount auth, @PathParam("username") @Username String username)
public Response setUsername(@Auth AuthenticatedAccount auth, @PathParam("username") String username)
throws RateLimitExceededException {
rateLimiters.getUsernameSetLimiter().validate(auth.getAccount().getUuid());
try {
accounts.setUsername(auth.getAccount(), username);
} catch (final UsernameNotAvailableException e) {
if (username == null || username.isEmpty()) {
return Response.status(Response.Status.BAD_REQUEST).build();
}
username = username.toLowerCase();
if (!username.matches("^[a-z_][a-z0-9_]+$")) {
return Response.status(Response.Status.BAD_REQUEST).build();
}
if (!usernames.put(auth.getAccount().getUuid(), username)) {
return Response.status(Response.Status.CONFLICT).build();
}
return Response.ok().build();
}
@HEAD
@Path("/account/{uuid}")
public Response accountExists(
@HeaderParam("X-Forwarded-For") final String forwardedFor,
@PathParam("uuid") final UUID uuid,
@Context HttpServletRequest request) throws RateLimitExceededException {
// Disallow clients from making authenticated requests to this endpoint
if (StringUtils.isNotBlank(request.getHeader("Authorization"))) {
throw new BadRequestException();
}
final String mostRecentProxy = ForwardedIpUtil.getMostRecentProxy(forwardedFor)
.orElseThrow(() -> new RateLimitExceededException(Duration.ofHours(1)));
rateLimiters.getCheckAccountExistenceLimiter().validate(mostRecentProxy);
final Status status = accounts.getByAccountIdentifier(uuid)
.or(() -> accounts.getByPhoneNumberIdentifier(uuid))
.isPresent() ? Status.OK : Status.NOT_FOUND;
return Response.status(status).build();
}
private void verifyRegistrationLock(final Account existingAccount, @Nullable final String clientRegistrationLock)
throws RateLimitExceededException, WebApplicationException {
@@ -14,7 +14,6 @@ import java.security.InvalidKeyException;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -71,8 +70,7 @@ public class CertificateController {
@Path("/group/{startRedemptionTime}/{endRedemptionTime}")
public GroupCredentials getAuthenticationCredentials(@Auth AuthenticatedAccount auth,
@PathParam("startRedemptionTime") int startRedemptionTime,
@PathParam("endRedemptionTime") int endRedemptionTime,
@QueryParam("identity") Optional<String> identityType) {
@PathParam("endRedemptionTime") int endRedemptionTime) {
if (startRedemptionTime > endRedemptionTime) {
throw new WebApplicationException(Response.Status.BAD_REQUEST);
}
@@ -85,13 +83,10 @@ public class CertificateController {
List<GroupCredentials.GroupCredential> credentials = new LinkedList<>();
final UUID identifier = identityType.map(String::toLowerCase).orElse("aci").equals("pni") ?
auth.getAccount().getPhoneNumberIdentifier() :
auth.getAccount().getUuid();
for (int i = startRedemptionTime; i <= endRedemptionTime; i++) {
credentials.add(new GroupCredentials.GroupCredential(
serverZkAuthOperations.issueAuthCredential(identifier, i).serialize(),
serverZkAuthOperations.issueAuthCredential(auth.getAccount().getUuid(), i)
.serialize(),
i));
}
@@ -41,7 +41,7 @@ import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.Device.DeviceCapabilities;
import org.whispersystems.textsecuregcm.storage.Keys;
import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.StoredVerificationCodeManager;
import org.whispersystems.textsecuregcm.util.Util;
@@ -57,14 +57,14 @@ public class DeviceController {
private final StoredVerificationCodeManager pendingDevices;
private final AccountsManager accounts;
private final MessagesManager messages;
private final Keys keys;
private final KeysDynamoDb keys;
private final RateLimiters rateLimiters;
private final Map<String, Integer> maxDeviceConfiguration;
public DeviceController(StoredVerificationCodeManager pendingDevices,
AccountsManager accounts,
MessagesManager messages,
Keys keys,
KeysDynamoDb keys,
RateLimiters rateLimiters,
Map<String, Integer> maxDeviceConfiguration)
{
@@ -168,7 +168,7 @@ public class DeviceController {
throw new WebApplicationException(Response.status(403).build());
}
Optional<Account> account = accounts.getByE164(number);
Optional<Account> account = accounts.get(number);
if (!account.isPresent()) {
throw new WebApplicationException(Response.status(403).build());
@@ -203,7 +203,7 @@ public class DeviceController {
device.setCreated(System.currentTimeMillis());
device.setCapabilities(accountAttributes.getCapabilities());
final Account updatedAccount = accounts.update(account.get(), a -> {
accounts.update(account.get(), a -> {
device.setId(a.getNextDeviceId());
messages.clear(a.getUuid(), device.getId());
a.addDevice(device);
@@ -211,7 +211,7 @@ public class DeviceController {
pendingDevices.remove(number);
return new DeviceResponse(updatedAccount.getUuid(), updatedAccount.getPhoneNumberIdentifier(), device.getId());
return new DeviceResponse(device.getId());
}
@Timed
@@ -1,49 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.controllers;
import com.codahale.metrics.annotation.Timed;
import io.dropwizard.auth.Auth;
import java.util.Base64;
import java.util.UUID;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentialGenerator;
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
import org.whispersystems.textsecuregcm.util.ByteUtil;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import org.whispersystems.textsecuregcm.util.Util;
@Path("/v2/directory")
public class DirectoryV2Controller {
private final ExternalServiceCredentialGenerator directoryServiceTokenGenerator;
public DirectoryV2Controller(ExternalServiceCredentialGenerator userTokenGenerator) {
this.directoryServiceTokenGenerator = userTokenGenerator;
}
@Timed
@GET
@Path("/auth")
@Produces(MediaType.APPLICATION_JSON)
public Response getAuthToken(@Auth AuthenticatedAccount auth) {
final UUID uuid = auth.getAccount().getUuid();
final String e164 = auth.getAccount().getNumber();
final long e164AsLong = Long.parseLong(e164, e164.indexOf('+'), e164.length() - 1, 10);
final byte[] uuidAndNumber = ByteUtil.combine(UUIDUtil.toBytes(uuid), Util.longToByteArray(e164AsLong));
final String username = Base64.getEncoder().encodeToString(uuidAndNumber);
final ExternalServiceCredentials credentials = directoryServiceTokenGenerator.generateFor(username);
return Response.ok().entity(credentials).build();
}
}
@@ -159,7 +159,7 @@ public class DonationController {
@Override
public boolean block() {
final Optional<Account> optionalAccount = accountsManager.getByAccountIdentifier(auth.getAccount().getUuid());
final Optional<Account> optionalAccount = accountsManager.get(auth.getAccount().getUuid());
optionalAccount.ifPresent(account -> {
accountsManager.update(account, a -> {
a.addBadge(clock, new AccountBadge(badgeId, receiptExpiration, request.isVisible()));
@@ -11,7 +11,6 @@ import io.dropwizard.auth.Auth;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -25,13 +24,11 @@ import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.whispersystems.textsecuregcm.auth.Anonymous;
import org.whispersystems.textsecuregcm.auth.AuthenticatedAccount;
import org.whispersystems.textsecuregcm.auth.ChangesDeviceEnabledState;
import org.whispersystems.textsecuregcm.auth.DisabledPermittedAuthenticatedAccount;
import org.whispersystems.textsecuregcm.auth.OptionalAccess;
import org.whispersystems.textsecuregcm.entities.PreKey;
@@ -40,11 +37,14 @@ import org.whispersystems.textsecuregcm.entities.PreKeyResponse;
import org.whispersystems.textsecuregcm.entities.PreKeyResponseItem;
import org.whispersystems.textsecuregcm.entities.PreKeyState;
import org.whispersystems.textsecuregcm.entities.SignedPreKey;
import org.whispersystems.textsecuregcm.limits.PreKeyRateLimiter;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeException;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.Keys;
import org.whispersystems.textsecuregcm.storage.KeysDynamoDb;
import org.whispersystems.textsecuregcm.util.Util;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@@ -52,26 +52,32 @@ import org.whispersystems.textsecuregcm.util.Util;
public class KeysController {
private final RateLimiters rateLimiters;
private final Keys keys;
private final KeysDynamoDb keysDynamoDb;
private final AccountsManager accounts;
private final PreKeyRateLimiter preKeyRateLimiter;
private final RateLimitChallengeManager rateLimitChallengeManager;
private static final String PREKEY_REQUEST_COUNTER_NAME = name(KeysController.class, "preKeyGet");
private static final String RATE_LIMITED_GET_PREKEYS_COUNTER_NAME = name(KeysController.class, "rateLimitedGetPreKeys");
private static final String SOURCE_COUNTRY_TAG_NAME = "sourceCountry";
private static final String INTERNATIONAL_TAG_NAME = "international";
public KeysController(RateLimiters rateLimiters, Keys keys, AccountsManager accounts) {
this.rateLimiters = rateLimiters;
this.keys = keys;
this.accounts = accounts;
public KeysController(RateLimiters rateLimiters, KeysDynamoDb keysDynamoDb, AccountsManager accounts,
PreKeyRateLimiter preKeyRateLimiter,
RateLimitChallengeManager rateLimitChallengeManager) {
this.rateLimiters = rateLimiters;
this.keysDynamoDb = keysDynamoDb;
this.accounts = accounts;
this.preKeyRateLimiter = preKeyRateLimiter;
this.rateLimitChallengeManager = rateLimitChallengeManager;
}
@GET
@Produces(MediaType.APPLICATION_JSON)
public PreKeyCount getStatus(@Auth final AuthenticatedAccount auth,
@QueryParam("identity") final Optional<String> identityType) {
int count = keys.getCount(getIdentifier(auth.getAccount(), identityType), auth.getAuthenticatedDevice().getId());
public PreKeyCount getStatus(@Auth AuthenticatedAccount auth) {
int count = keysDynamoDb.getCount(auth.getAccount(), auth.getAuthenticatedDevice().getId());
if (count > 0) {
count = count - 1;
@@ -83,10 +89,7 @@ public class KeysController {
@Timed
@PUT
@Consumes(MediaType.APPLICATION_JSON)
@ChangesDeviceEnabledState
public void setKeys(@Auth final DisabledPermittedAuthenticatedAccount disabledPermittedAuth,
@Valid final PreKeyState preKeys,
@QueryParam("identity") final Optional<String> identityType) {
public void setKeys(@Auth DisabledPermittedAuthenticatedAccount disabledPermittedAuth, @Valid PreKeyState preKeys) {
Account account = disabledPermittedAuth.getAccount();
Device device = disabledPermittedAuth.getAuthenticatedDevice();
boolean updateAccount = false;
@@ -99,27 +102,14 @@ public class KeysController {
updateAccount = true;
}
final boolean usePhoneNumberIdentity = usePhoneNumberIdentity(identityType);
if (updateAccount) {
account = accounts.update(account, a -> {
a.getDevice(device.getId()).ifPresent(d -> {
if (usePhoneNumberIdentity) {
d.setPhoneNumberIdentitySignedPreKey(preKeys.getSignedPreKey());
} else {
d.setSignedPreKey(preKeys.getSignedPreKey());
}
});
if (usePhoneNumberIdentity) {
a.setPhoneNumberIdentityKey(preKeys.getIdentityKey());
} else {
a.setIdentityKey(preKeys.getIdentityKey());
}
a.getDevice(device.getId()).ifPresent(d -> d.setSignedPreKey(preKeys.getSignedPreKey()));
a.setIdentityKey(preKeys.getIdentityKey());
});
}
keys.store(getIdentifier(account, identityType), device.getId(), preKeys.getPreKeys());
keysDynamoDb.store(account, device.getId(), preKeys.getPreKeys());
}
@Timed
@@ -131,7 +121,7 @@ public class KeysController {
@PathParam("identifier") UUID targetUuid,
@PathParam("device_id") String deviceId,
@HeaderParam("User-Agent") String userAgent)
throws RateLimitExceededException {
throws RateLimitExceededException, RateLimitChallengeException, ServerRejectedException {
if (!auth.isPresent() && !accessKey.isPresent()) {
throw new WebApplicationException(Response.Status.UNAUTHORIZED);
@@ -139,19 +129,14 @@ public class KeysController {
final Optional<Account> account = auth.map(AuthenticatedAccount::getAccount);
final Account target;
{
final Optional<Account> maybeTarget = accounts.getByAccountIdentifier(targetUuid)
.or(() -> accounts.getByPhoneNumberIdentifier(targetUuid));
Optional<Account> target = accounts.get(targetUuid);
OptionalAccess.verify(account, accessKey, target, deviceId);
OptionalAccess.verify(account, accessKey, maybeTarget, deviceId);
target = maybeTarget.orElseThrow();
}
assert (target.isPresent());
{
final String sourceCountryCode = account.map(a -> Util.getCountryCode(a.getNumber())).orElse("0");
final String targetCountryCode = Util.getCountryCode(target.getNumber());
final String targetCountryCode = target.map(a -> Util.getCountryCode(a.getNumber())).orElseThrow();
Metrics.counter(PREKEY_REQUEST_COUNTER_NAME, Tags.of(
SOURCE_COUNTRY_TAG_NAME, sourceCountryCode,
@@ -161,18 +146,33 @@ public class KeysController {
if (account.isPresent()) {
rateLimiters.getPreKeysLimiter().validate(
account.get().getUuid() + "." + auth.get().getAuthenticatedDevice().getId() + "__" + targetUuid
account.get().getUuid() + "." + auth.get().getAuthenticatedDevice().getId() + "__" + target.get().getUuid()
+ "." + deviceId);
try {
preKeyRateLimiter.validate(account.get());
} catch (RateLimitExceededException e) {
final boolean legacyClient = rateLimitChallengeManager.isClientBelowMinimumVersion(userAgent);
Metrics.counter(RATE_LIMITED_GET_PREKEYS_COUNTER_NAME,
SOURCE_COUNTRY_TAG_NAME, Util.getCountryCode(account.get().getNumber()),
"legacyClient", String.valueOf(legacyClient))
.increment();
if (legacyClient) {
throw new ServerRejectedException();
}
throw new RateLimitChallengeException(account.get(), e.getRetryDuration());
}
}
final boolean usePhoneNumberIdentity = target.getPhoneNumberIdentifier().equals(targetUuid);
Map<Long, PreKey> preKeysByDeviceId = getLocalKeys(target, deviceId, usePhoneNumberIdentity);
Map<Long, PreKey> preKeysByDeviceId = getLocalKeys(target.get(), deviceId);
List<PreKeyResponseItem> responseItems = new LinkedList<>();
for (Device device : target.getDevices()) {
for (Device device : target.get().getDevices()) {
if (device.isEnabled() && (deviceId.equals("*") || device.getId() == Long.parseLong(deviceId))) {
SignedPreKey signedPreKey = usePhoneNumberIdentity ? device.getPhoneNumberIdentitySignedPreKey() : device.getSignedPreKey();
SignedPreKey signedPreKey = device.getSignedPreKey();
PreKey preKey = preKeysByDeviceId.get(device.getId());
if (signedPreKey != null || preKey != null) {
@@ -181,81 +181,48 @@ public class KeysController {
}
}
final String identityKey = usePhoneNumberIdentity ? target.getPhoneNumberIdentityKey() : target.getIdentityKey();
if (responseItems.isEmpty()) return Response.status(404).build();
else return Response.ok().entity(new PreKeyResponse(identityKey, responseItems)).build();
else return Response.ok().entity(new PreKeyResponse(target.get().getIdentityKey(), responseItems)).build();
}
@Timed
@PUT
@Path("/signed")
@Consumes(MediaType.APPLICATION_JSON)
@ChangesDeviceEnabledState
public void setSignedKey(@Auth final AuthenticatedAccount auth,
@Valid final SignedPreKey signedPreKey,
@QueryParam("identity") final Optional<String> identityType) {
public void setSignedKey(@Auth AuthenticatedAccount auth, @Valid SignedPreKey signedPreKey) {
Device device = auth.getAuthenticatedDevice();
accounts.updateDevice(auth.getAccount(), device.getId(), d -> {
if (usePhoneNumberIdentity(identityType)) {
d.setPhoneNumberIdentitySignedPreKey(signedPreKey);
} else {
d.setSignedPreKey(signedPreKey);
}
});
accounts.updateDevice(auth.getAccount(), device.getId(), d -> d.setSignedPreKey(signedPreKey));
}
@Timed
@GET
@Path("/signed")
@Produces(MediaType.APPLICATION_JSON)
public Optional<SignedPreKey> getSignedKey(@Auth final AuthenticatedAccount auth,
@QueryParam("identity") final Optional<String> identityType) {
public Optional<SignedPreKey> getSignedKey(@Auth AuthenticatedAccount auth) {
Device device = auth.getAuthenticatedDevice();
SignedPreKey signedPreKey = usePhoneNumberIdentity(identityType) ?
device.getPhoneNumberIdentitySignedPreKey() : device.getSignedPreKey();
SignedPreKey signedPreKey = device.getSignedPreKey();
return Optional.ofNullable(signedPreKey);
}
private static boolean usePhoneNumberIdentity(final Optional<String> identityType) {
return "pni".equals(identityType.map(String::toLowerCase).orElse("aci"));
}
private static UUID getIdentifier(final Account account, final Optional<String> identityType) {
return usePhoneNumberIdentity(identityType) ?
account.getPhoneNumberIdentifier() :
account.getUuid();
}
private Map<Long, PreKey> getLocalKeys(Account destination, String deviceIdSelector, final boolean usePhoneNumberIdentity) {
final Map<Long, PreKey> preKeys;
final UUID identifier = usePhoneNumberIdentity ?
destination.getPhoneNumberIdentifier() :
destination.getUuid();
if (deviceIdSelector.equals("*")) {
preKeys = new HashMap<>();
for (final Device device : destination.getDevices()) {
keys.take(identifier, device.getId()).ifPresent(preKey -> preKeys.put(device.getId(), preKey));
}
if (signedPreKey != null) {
return Optional.of(signedPreKey);
} else {
try {
long deviceId = Long.parseLong(deviceIdSelector);
preKeys = keys.take(identifier, deviceId)
.map(preKey -> Map.of(deviceId, preKey))
.orElse(Collections.emptyMap());
} catch (NumberFormatException e) {
throw new WebApplicationException(Response.status(422).build());
}
return Optional.empty();
}
}
return preKeys;
private Map<Long, PreKey> getLocalKeys(Account destination, String deviceIdSelector) {
try {
if (deviceIdSelector.equals("*")) {
return keysDynamoDb.take(destination);
}
long deviceId = Long.parseLong(deviceIdSelector);
return keysDynamoDb.take(destination, deviceId)
.map(preKey -> Map.of(deviceId, preKey))
.orElse(Collections.emptyMap());
} catch (NumberFormatException e) {
throw new WebApplicationException(Response.status(422).build());
}
}
}
@@ -6,10 +6,14 @@ package org.whispersystems.textsecuregcm.controllers;
import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import com.codahale.metrics.annotation.Timed;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
import com.vdurmont.semver4j.Semver;
import io.dropwizard.auth.Auth;
import io.dropwizard.util.DataSize;
import io.micrometer.core.instrument.Counter;
@@ -38,7 +42,6 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.validation.Valid;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
@@ -75,7 +78,9 @@ import org.whispersystems.textsecuregcm.entities.SendMessageResponse;
import org.whispersystems.textsecuregcm.entities.SendMultiRecipientMessageResponse;
import org.whispersystems.textsecuregcm.entities.StaleDevices;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeException;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.limits.UnsealedSenderRateLimiter;
import org.whispersystems.textsecuregcm.metrics.UserAgentTagUtil;
import org.whispersystems.textsecuregcm.providers.MultiRecipientMessageProvider;
import org.whispersystems.textsecuregcm.push.ApnFallbackManager;
@@ -88,11 +93,10 @@ import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.Device;
import org.whispersystems.textsecuregcm.storage.MessagesManager;
import org.whispersystems.textsecuregcm.storage.ReportMessageManager;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.Pair;
import org.whispersystems.textsecuregcm.util.Util;
import org.whispersystems.textsecuregcm.util.ua.ClientPlatform;
import org.whispersystems.textsecuregcm.util.ua.UnrecognizedUserAgentException;
import org.whispersystems.textsecuregcm.util.ua.UserAgent;
import org.whispersystems.textsecuregcm.util.ua.UserAgentUtil;
import org.whispersystems.textsecuregcm.websocket.WebSocketConnection;
@@ -100,24 +104,26 @@ import org.whispersystems.textsecuregcm.websocket.WebSocketConnection;
@Path("/v1/messages")
public class MessageController {
private static final Logger logger = LoggerFactory.getLogger(MessageController.class);
private final Logger logger = LoggerFactory.getLogger(MessageController.class);
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Meter unidentifiedMeter = metricRegistry.meter(name(getClass(), "delivery", "unidentified"));
private final Meter identifiedMeter = metricRegistry.meter(name(getClass(), "delivery", "identified" ));
private final Meter rejectOver256kibMessageMeter = metricRegistry.meter(name(getClass(), "rejectOver256kibMessage"));
private final Timer sendMessageInternalTimer = metricRegistry.timer(name(getClass(), "sendMessageInternal"));
private final Timer sendCommonMessageInternalTimer = metricRegistry.timer(name(getClass(), "sendCommonMessageInternal"));
private final Histogram outgoingMessageListSizeHistogram = metricRegistry.histogram(name(getClass(), "outgoingMessageListSize"));
private final RateLimiters rateLimiters;
private final MessageSender messageSender;
private final ReceiptSender receiptSender;
private final AccountsManager accountsManager;
private final MessagesManager messagesManager;
private final UnsealedSenderRateLimiter unsealedSenderRateLimiter;
private final ApnFallbackManager apnFallbackManager;
private final RateLimitChallengeManager rateLimitChallengeManager;
private final ReportMessageManager reportMessageManager;
private final ExecutorService multiRecipientMessageExecutor;
@VisibleForTesting
static final Semver FIRST_IOS_VERSION_WITH_INCORRECT_ENVELOPE_TYPE = new Semver("5.22.0");
@VisibleForTesting
static final Semver IOS_VERSION_WITH_FIXED_ENVELOPE_TYPE = new Semver("5.25.0");
private static final String REJECT_OVERSIZE_MESSAGE_COUNTER = name(MessageController.class, "rejectOversizeMessage");
private static final String LEGACY_MESSAGE_SENT_COUNTER = name(MessageController.class, "legacyMessageSent");
private static final String SENT_MESSAGE_COUNTER_NAME = name(MessageController.class, "sentMessages");
private static final String CONTENT_SIZE_DISTRIBUTION_NAME = name(MessageController.class, "messageContentSize");
@@ -129,10 +135,6 @@ public class MessageController {
private static final String SENDER_COUNTRY_TAG_NAME = "senderCountry";
private static final String RATE_LIMIT_REASON_TAG_NAME = "rateLimitReason";
private static final String SENDER_TYPE_IDENTIFIED = "identified";
private static final String SENDER_TYPE_UNIDENTIFIED = "unidentified";
private static final String SENDER_TYPE_SELF = "self";
private static final long MAX_MESSAGE_SIZE = DataSize.kibibytes(256).toBytes();
public MessageController(
@@ -141,7 +143,9 @@ public class MessageController {
ReceiptSender receiptSender,
AccountsManager accountsManager,
MessagesManager messagesManager,
UnsealedSenderRateLimiter unsealedSenderRateLimiter,
ApnFallbackManager apnFallbackManager,
RateLimitChallengeManager rateLimitChallengeManager,
ReportMessageManager reportMessageManager,
@Nonnull ExecutorService multiRecipientMessageExecutor) {
this.rateLimiters = rateLimiters;
@@ -149,7 +153,9 @@ public class MessageController {
this.receiptSender = receiptSender;
this.accountsManager = accountsManager;
this.messagesManager = messagesManager;
this.unsealedSenderRateLimiter = unsealedSenderRateLimiter;
this.apnFallbackManager = apnFallbackManager;
this.rateLimitChallengeManager = rateLimitChallengeManager;
this.reportMessageManager = reportMessageManager;
this.multiRecipientMessageExecutor = Objects.requireNonNull(multiRecipientMessageExecutor);
}
@@ -174,14 +180,14 @@ public class MessageController {
final String senderType;
if (source.isPresent()) {
if (source.get().getAccount().isIdentifiedBy(destinationUuid)) {
senderType = SENDER_TYPE_SELF;
} else {
senderType = SENDER_TYPE_IDENTIFIED;
}
if (source.isPresent() && !source.get().getAccount().getUuid().equals(destinationUuid)) {
identifiedMeter.mark();
senderType = "identified";
} else if (source.isEmpty()) {
unidentifiedMeter.mark();
senderType = "unidentified";
} else {
senderType = SENDER_TYPE_UNIDENTIFIED;
senderType = "self";
}
for (final IncomingMessage message : messages.getMessages()) {
@@ -198,19 +204,18 @@ public class MessageController {
Metrics.summary(CONTENT_SIZE_DISTRIBUTION_NAME, Tags.of(UserAgentTagUtil.getPlatformTag(userAgent))).record(contentLength);
if (contentLength > MAX_MESSAGE_SIZE) {
Metrics.counter(REJECT_OVERSIZE_MESSAGE_COUNTER, Tags.of(UserAgentTagUtil.getPlatformTag(userAgent))).increment();
rejectOver256kibMessageMeter.mark();
return Response.status(Response.Status.REQUEST_ENTITY_TOO_LARGE).build();
}
}
try {
boolean isSyncMessage = source.isPresent() && source.get().getAccount().isIdentifiedBy(destinationUuid);
boolean isSyncMessage = source.isPresent() && source.get().getAccount().getUuid().equals(destinationUuid);
Optional<Account> destination;
if (!isSyncMessage) {
destination = accountsManager.getByAccountIdentifier(destinationUuid)
.or(() -> accountsManager.getByPhoneNumberIdentifier(destinationUuid));
destination = accountsManager.get(destinationUuid);
} else {
destination = source.map(AuthenticatedAccount::getAccount);
}
@@ -218,7 +223,7 @@ public class MessageController {
OptionalAccess.verify(source.map(AuthenticatedAccount::getAccount), accessKey, destination);
assert (destination.isPresent());
if (source.isPresent() && !source.get().getAccount().isIdentifiedBy(destinationUuid)) {
if (source.isPresent() && !source.get().getAccount().getUuid().equals(destinationUuid)) {
final String senderCountryCode = Util.getCountryCode(source.get().getAccount().getNumber());
try {
@@ -230,6 +235,24 @@ public class MessageController {
throw e;
}
try {
unsealedSenderRateLimiter.validate(source.get().getAccount(), destination.get());
} catch (final RateLimitExceededException e) {
final boolean legacyClient = rateLimitChallengeManager.isClientBelowMinimumVersion(userAgent);
final String rateLimitReason = legacyClient ? "unsealedSenderCardinality" : "challengeIssued";
Metrics.counter(RATE_LIMITED_MESSAGE_COUNTER_NAME,
SENDER_COUNTRY_TAG_NAME, senderCountryCode,
RATE_LIMIT_REASON_TAG_NAME, rateLimitReason).increment();
if (legacyClient) {
throw e;
}
throw new RateLimitChallengeException(source.get().getAccount(), e.getRetryDuration());
}
}
validateCompleteDeviceList(destination.get(), messages.getMessages(), isSyncMessage,
@@ -245,8 +268,8 @@ public class MessageController {
if (destinationDevice.isPresent()) {
Metrics.counter(SENT_MESSAGE_COUNTER_NAME, tags).increment();
sendMessage(source, destination.get(), destinationDevice.get(), destinationUuid, messages.getTimestamp(),
messages.isOnline(), incomingMessage, userAgent);
sendMessage(source, destination.get(), destinationDevice.get(), messages.getTimestamp(), messages.isOnline(),
incomingMessage);
}
}
@@ -282,11 +305,13 @@ public class MessageController {
@QueryParam("ts") long timestamp,
@Valid MultiRecipientMessage multiRecipientMessage) {
unidentifiedMeter.mark(multiRecipientMessage.getRecipients().length);
Map<UUID, Account> uuidToAccountMap = Arrays.stream(multiRecipientMessage.getRecipients())
.map(Recipient::getUuid)
.distinct()
.collect(Collectors.toUnmodifiableMap(Function.identity(), uuid -> {
Optional<Account> account = accountsManager.getByAccountIdentifier(uuid);
Optional<Account> account = accountsManager.get(uuid);
if (account.isEmpty()) {
throw new WebApplicationException(Status.NOT_FOUND);
}
@@ -337,21 +362,20 @@ public class MessageController {
.build();
}
List<Tag> tags = List.of(
UserAgentTagUtil.getPlatformTag(userAgent),
Tag.of(EPHEMERAL_TAG_NAME, String.valueOf(online)),
Tag.of(SENDER_TYPE_TAG_NAME, "unidentified"));
List<UUID> uuids404 = Collections.synchronizedList(new ArrayList<>());
final Counter counter = Metrics.counter(SENT_MESSAGE_COUNTER_NAME, tags);
try {
final Counter sentMessageCounter = Metrics.counter(SENT_MESSAGE_COUNTER_NAME, Tags.of(
UserAgentTagUtil.getPlatformTag(userAgent),
Tag.of(EPHEMERAL_TAG_NAME, String.valueOf(online)),
Tag.of(SENDER_TYPE_TAG_NAME, SENDER_TYPE_UNIDENTIFIED)));
multiRecipientMessageExecutor.invokeAll(Arrays.stream(multiRecipientMessage.getRecipients())
.map(recipient -> (Callable<Void>) () -> {
Account destinationAccount = uuidToAccountMap.get(recipient.getUuid());
// we asserted this must exist in validateCompleteDeviceList
Device destinationDevice = destinationAccount.getDevice(recipient.getDeviceId()).orElseThrow();
sentMessageCounter.increment();
counter.increment();
try {
sendMessage(destinationAccount, destinationDevice, timestamp, online, recipient,
multiRecipientMessage.getCommonPayload());
@@ -420,6 +444,8 @@ public class MessageController {
userAgent,
false);
outgoingMessageListSizeHistogram.update(outgoingMessages.getMessages().size());
{
String platform;
@@ -477,7 +503,7 @@ public class MessageController {
public Response reportMessage(@Auth AuthenticatedAccount auth, @PathParam("sourceNumber") String sourceNumber,
@PathParam("messageGuid") UUID messageGuid) {
reportMessageManager.report(sourceNumber, messageGuid, auth.getAccount().getUuid());
reportMessageManager.report(sourceNumber, messageGuid);
return Response.status(Status.ACCEPTED)
.build();
@@ -486,56 +512,33 @@ public class MessageController {
private void sendMessage(Optional<AuthenticatedAccount> source,
Account destinationAccount,
Device destinationDevice,
UUID destinationUuid,
long timestamp,
boolean online,
IncomingMessage incomingMessage,
String userAgentString)
IncomingMessage incomingMessage)
throws NoSuchUserException {
try {
try (final Timer.Context ignored = sendMessageInternalTimer.time()) {
Optional<byte[]> messageBody = getMessageBody(incomingMessage);
Optional<byte[]> messageContent = getMessageContent(incomingMessage);
Envelope.Builder messageBuilder = Envelope.newBuilder();
int envelopeTypeNumber = incomingMessage.getType();
// Some versions of the iOS app incorrectly use the reserved envelope type 7 for PLAINTEXT_CONTENT instead of type
// 8. This check can be removed safely after 2022-03-01.
if (envelopeTypeNumber == 7) {
try {
final UserAgent userAgent = UserAgentUtil.parseUserAgentString(userAgentString);
if (userAgent.getPlatform() == ClientPlatform.IOS &&
FIRST_IOS_VERSION_WITH_INCORRECT_ENVELOPE_TYPE.isLowerThanOrEqualTo(userAgent.getVersion()) &&
userAgent.getVersion().isLowerThan(IOS_VERSION_WITH_FIXED_ENVELOPE_TYPE)) {
envelopeTypeNumber = Type.PLAINTEXT_CONTENT.getNumber();
}
} catch (final UnrecognizedUserAgentException ignored2) {
}
}
final Envelope.Type envelopeType = Envelope.Type.forNumber(envelopeTypeNumber);
if (envelopeType == null) {
logger.warn("Received bad envelope type {} from {}", incomingMessage.getType(), userAgentString);
throw new BadRequestException();
}
messageBuilder.setType(envelopeType)
messageBuilder.setType(Envelope.Type.forNumber(incomingMessage.getType()))
.setTimestamp(timestamp == 0 ? System.currentTimeMillis() : timestamp)
.setServerTimestamp(System.currentTimeMillis())
.setDestinationUuid(destinationUuid.toString());
.setServerTimestamp(System.currentTimeMillis());
source.ifPresent(authenticatedAccount ->
messageBuilder.setSource(authenticatedAccount.getAccount().getNumber())
.setSourceUuid(authenticatedAccount.getAccount().getUuid().toString())
.setSourceDevice((int) authenticatedAccount.getAuthenticatedDevice().getId()));
if (source.isPresent()) {
messageBuilder.setSource(source.get().getAccount().getNumber())
.setSourceUuid(source.get().getAccount().getUuid().toString())
.setSourceDevice((int) source.get().getAuthenticatedDevice().getId());
}
messageBody.ifPresent(bytes -> {
if (messageBody.isPresent()) {
Metrics.counter(LEGACY_MESSAGE_SENT_COUNTER).increment();
messageBuilder.setLegacyMessage(ByteString.copyFrom(messageBody.get()));
});
}
messageContent.ifPresent(bytes -> messageBuilder.setContent(ByteString.copyFrom(bytes)));
if (messageContent.isPresent()) {
messageBuilder.setContent(ByteString.copyFrom(messageContent.get()));
}
messageSender.sendMessage(destinationAccount, destinationDevice, messageBuilder.build(), online);
} catch (NotPushRegisteredException e) {
@@ -544,13 +547,9 @@ public class MessageController {
}
}
private void sendMessage(Account destinationAccount,
Device destinationDevice,
long timestamp,
boolean online,
Recipient recipient,
byte[] commonPayload) throws NoSuchUserException {
try {
private void sendMessage(Account destinationAccount, Device destinationDevice, long timestamp, boolean online,
Recipient recipient, byte[] commonPayload) throws NoSuchUserException {
try (final Timer.Context ignored = sendCommonMessageInternalTimer.time()) {
Envelope.Builder messageBuilder = Envelope.newBuilder();
long serverTimestamp = System.currentTimeMillis();
byte[] recipientKeyMaterial = recipient.getPerRecipientKeyMaterial();
@@ -564,8 +563,7 @@ public class MessageController {
.setType(Type.UNIDENTIFIED_SENDER)
.setTimestamp(timestamp == 0 ? serverTimestamp : timestamp)
.setServerTimestamp(serverTimestamp)
.setContent(ByteString.copyFrom(payload))
.setDestinationUuid(destinationAccount.getUuid().toString());
.setContent(ByteString.copyFrom(payload));
messageSender.sendMessage(destinationAccount, destinationDevice, messageBuilder.build(), online);
} catch (NotPushRegisteredException e) {
@@ -5,12 +5,8 @@
package org.whispersystems.textsecuregcm.controllers;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import com.codahale.metrics.annotation.Timed;
import io.dropwizard.auth.Auth;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
import java.security.SecureRandom;
import java.time.Clock;
import java.time.Duration;
@@ -23,15 +19,13 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.validation.Valid;
import javax.validation.valueextraction.Unwrapping;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.ForbiddenException;
import javax.ws.rs.GET;
import javax.ws.rs.HeaderParam;
import javax.ws.rs.PUT;
@@ -52,7 +46,6 @@ import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang3.StringUtils;
import org.signal.zkgroup.InvalidInputException;
import org.signal.zkgroup.VerificationFailedException;
import org.signal.zkgroup.profiles.PniCredentialResponse;
import org.signal.zkgroup.profiles.ProfileKeyCommitment;
import org.signal.zkgroup.profiles.ProfileKeyCredentialRequest;
import org.signal.zkgroup.profiles.ProfileKeyCredentialResponse;
@@ -72,7 +65,6 @@ import org.whispersystems.textsecuregcm.entities.Profile;
import org.whispersystems.textsecuregcm.entities.ProfileAvatarUploadAttributes;
import org.whispersystems.textsecuregcm.entities.UserCapabilities;
import org.whispersystems.textsecuregcm.limits.RateLimiters;
import org.whispersystems.textsecuregcm.metrics.UserAgentTagUtil;
import org.whispersystems.textsecuregcm.s3.PolicySigner;
import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator;
import org.whispersystems.textsecuregcm.storage.Account;
@@ -80,9 +72,11 @@ import org.whispersystems.textsecuregcm.storage.AccountBadge;
import org.whispersystems.textsecuregcm.storage.AccountsManager;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.storage.ProfilesManager;
import org.whispersystems.textsecuregcm.storage.UsernamesManager;
import org.whispersystems.textsecuregcm.storage.VersionedProfile;
import org.whispersystems.textsecuregcm.util.ExactlySize;
import org.whispersystems.textsecuregcm.util.Pair;
import org.whispersystems.textsecuregcm.util.Util;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
@@ -96,6 +90,7 @@ public class ProfileController {
private final RateLimiters rateLimiters;
private final ProfilesManager profilesManager;
private final AccountsManager accountsManager;
private final UsernamesManager usernamesManager;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
private final ProfileBadgeConverter profileBadgeConverter;
private final Map<String, BadgeConfiguration> badgeConfigurationMap;
@@ -107,16 +102,12 @@ public class ProfileController {
private final S3Client s3client;
private final String bucket;
private static final String PROFILE_KEY_CREDENTIAL_TYPE = "profileKey";
private static final String PNI_CREDENTIAL_TYPE = "pni";
private static final String LEGACY_GET_PROFILE_COUNTER_NAME = name(ProfileController.class, "legacyGetProfileByPlatform");
public ProfileController(
Clock clock,
RateLimiters rateLimiters,
AccountsManager accountsManager,
ProfilesManager profilesManager,
UsernamesManager usernamesManager,
DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
ProfileBadgeConverter profileBadgeConverter,
BadgesConfiguration badgesConfiguration,
@@ -129,6 +120,7 @@ public class ProfileController {
this.rateLimiters = rateLimiters;
this.accountsManager = accountsManager;
this.profilesManager = profilesManager;
this.usernamesManager = usernamesManager;
this.dynamicConfigurationManager = dynamicConfigurationManager;
this.profileBadgeConverter = profileBadgeConverter;
this.badgeConfigurationMap = badgesConfiguration.getBadges().stream().collect(Collectors.toMap(
@@ -145,14 +137,13 @@ public class ProfileController {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public Response setProfile(@Auth AuthenticatedAccount auth, @Valid CreateProfileRequest request) {
if (StringUtils.isNotBlank(request.getPaymentAddress())) {
final boolean hasDisallowedPrefix =
dynamicConfigurationManager.getConfiguration().getPaymentsConfiguration().getDisallowedPrefixes().stream()
.anyMatch(prefix -> auth.getAccount().getNumber().startsWith(prefix));
final Set<String> allowedPaymentsCountryCodes =
dynamicConfigurationManager.getConfiguration().getPaymentsConfiguration().getAllowedCountryCodes();
if (hasDisallowedPrefix) {
return Response.status(Status.FORBIDDEN).build();
}
if (StringUtils.isNotBlank(request.getPaymentAddress()) &&
!allowedPaymentsCountryCodes.contains(Util.getCountryCode(auth.getAccount().getNumber()))) {
return Response.status(Status.FORBIDDEN).build();
}
Optional<VersionedProfile> currentProfile = profilesManager.get(auth.getAccount().getUuid(), request.getVersion());
@@ -190,9 +181,8 @@ public class ProfileController {
response = Optional.of(generateAvatarUploadForm(avatar));
}
List<AccountBadge> updatedBadges = request.getBadges()
.map(badges -> mergeBadgeIdsWithExistingAccountBadges(badges, auth.getAccount().getBadges()))
.orElseGet(() -> auth.getAccount().getBadges());
List<AccountBadge> updatedBadges = mergeBadgeIdsWithExistingAccountBadges(
request.getBadges(), auth.getAccount().getBadges());
accountsManager.update(auth.getAccount(), a -> {
a.setProfileName(request.getName());
@@ -212,7 +202,7 @@ public class ProfileController {
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/{uuid}/{version}")
public Profile getProfile(
public Optional<Profile> getProfile(
@Auth Optional<AuthenticatedAccount> auth,
@HeaderParam(OptionalAccess.UNIDENTIFIED) Optional<Anonymous> accessKey,
@Context ContainerRequestContext containerRequestContext,
@@ -221,111 +211,87 @@ public class ProfileController {
throws RateLimitExceededException {
return getVersionedProfile(auth.map(AuthenticatedAccount::getAccount), accessKey,
getAcceptableLanguagesForRequest(containerRequestContext), uuid,
version, Optional.empty(), Optional.empty());
version, Optional.empty());
}
@Timed
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/{uuid}/{version}/{credentialRequest}")
public Profile getProfile(
public Optional<Profile> getProfile(
@Auth Optional<AuthenticatedAccount> auth,
@HeaderParam(OptionalAccess.UNIDENTIFIED) Optional<Anonymous> accessKey,
@Context ContainerRequestContext containerRequestContext,
@PathParam("uuid") UUID uuid,
@PathParam("version") String version,
@PathParam("credentialRequest") String credentialRequest,
@QueryParam("credentialType") @DefaultValue(PROFILE_KEY_CREDENTIAL_TYPE) String credentialType)
@PathParam("credentialRequest") String credentialRequest)
throws RateLimitExceededException {
return getVersionedProfile(auth.map(AuthenticatedAccount::getAccount), accessKey,
getAcceptableLanguagesForRequest(containerRequestContext), uuid,
version, Optional.of(credentialRequest), Optional.of(credentialType));
version, Optional.of(credentialRequest));
}
private Profile getVersionedProfile(
private Optional<Profile> getVersionedProfile(
Optional<Account> requestAccount,
Optional<Anonymous> accessKey,
List<Locale> acceptableLanguages,
UUID uuid,
String version,
Optional<String> credentialRequest,
Optional<String> credentialType)
Optional<String> credentialRequest)
throws RateLimitExceededException {
if (requestAccount.isEmpty() && accessKey.isEmpty()) {
throw new WebApplicationException(Response.Status.UNAUTHORIZED);
}
boolean isSelf = false;
if (requestAccount.isPresent()) {
UUID authedUuid = requestAccount.get().getUuid();
rateLimiters.getProfileLimiter().validate(authedUuid);
isSelf = uuid.equals(authedUuid);
}
Optional<Account> accountProfile = accountsManager.getByAccountIdentifier(uuid);
OptionalAccess.verify(requestAccount, accessKey, accountProfile);
assert accountProfile.isPresent();
Optional<String> username = accountProfile.flatMap(Account::getUsername);
Optional<VersionedProfile> profile = profilesManager.get(uuid, version);
String name = profile.map(VersionedProfile::getName).orElse(accountProfile.get().getProfileName());
String about = profile.map(VersionedProfile::getAbout).orElse(null);
String aboutEmoji = profile.map(VersionedProfile::getAboutEmoji).orElse(null);
String avatar = profile.map(VersionedProfile::getAvatar).orElse(accountProfile.get().getAvatar());
Optional<String> currentProfileVersion = accountProfile.get().getCurrentProfileVersion();
// Allow requests where either the version matches the latest version on Account or the latest version on Account
// is empty to read the payment address.
final String paymentAddress = profile
.filter(p -> currentProfileVersion.map(v -> v.equals(version)).orElse(true))
.map(VersionedProfile::getPaymentAddress)
.orElse(null);
final ProfileKeyCredentialResponse profileKeyCredentialResponse;
final PniCredentialResponse pniCredentialResponse;
if (credentialRequest.isPresent() && credentialType.isPresent() && profile.isPresent()) {
if (PNI_CREDENTIAL_TYPE.equals(credentialType.get())) {
if (!isSelf) {
throw new ForbiddenException();
}
profileKeyCredentialResponse = null;
pniCredentialResponse = getPniCredential(credentialRequest.get(),
profile.get(),
requestAccount.get().getUuid(),
requestAccount.get().getPhoneNumberIdentifier());
} else if (PROFILE_KEY_CREDENTIAL_TYPE.equals(credentialType.get())) {
profileKeyCredentialResponse = getProfileCredential(credentialRequest.get(),
profile.get(),
uuid);
pniCredentialResponse = null;
} else {
throw new BadRequestException();
try {
if (requestAccount.isEmpty() && accessKey.isEmpty()) {
throw new WebApplicationException(Response.Status.UNAUTHORIZED);
}
} else {
profileKeyCredentialResponse = null;
pniCredentialResponse = null;
}
return new Profile(
name,
about,
aboutEmoji,
avatar,
paymentAddress,
accountProfile.get().getIdentityKey(),
UnidentifiedAccessChecksum.generateFor(accountProfile.get().getUnidentifiedAccessKey()),
accountProfile.get().isUnrestrictedUnidentifiedAccess(),
UserCapabilities.createForAccount(accountProfile.get()),
username.orElse(null),
null,
profileBadgeConverter.convert(acceptableLanguages, accountProfile.get().getBadges(), isSelf),
profileKeyCredentialResponse,
pniCredentialResponse);
boolean isSelf = false;
if (requestAccount.isPresent()) {
UUID authedUuid = requestAccount.get().getUuid();
rateLimiters.getProfileLimiter().validate(authedUuid);
isSelf = uuid.equals(authedUuid);
}
Optional<Account> accountProfile = accountsManager.get(uuid);
OptionalAccess.verify(requestAccount, accessKey, accountProfile);
assert(accountProfile.isPresent());
Optional<String> username = usernamesManager.get(accountProfile.get().getUuid());
Optional<VersionedProfile> profile = profilesManager.get(uuid, version);
String name = profile.map(VersionedProfile::getName).orElse(accountProfile.get().getProfileName());
String about = profile.map(VersionedProfile::getAbout).orElse(null);
String aboutEmoji = profile.map(VersionedProfile::getAboutEmoji).orElse(null);
String avatar = profile.map(VersionedProfile::getAvatar).orElse(accountProfile.get().getAvatar());
Optional<String> currentProfileVersion = accountProfile.get().getCurrentProfileVersion();
// Allow requests where either the version matches the latest version on Account or the latest version on Account
// is empty to read the payment address.
final String paymentAddress = profile
.filter(p -> currentProfileVersion.map(v -> v.equals(version)).orElse(true))
.map(VersionedProfile::getPaymentAddress)
.orElse(null);
Optional<ProfileKeyCredentialResponse> credential = getProfileCredential(credentialRequest, profile, uuid);
return Optional.of(new Profile(
name,
about,
aboutEmoji,
avatar,
paymentAddress,
accountProfile.get().getIdentityKey(),
UnidentifiedAccessChecksum.generateFor(accountProfile.get().getUnidentifiedAccessKey()),
accountProfile.get().isUnrestrictedUnidentifiedAccess(),
UserCapabilities.createForAccount(accountProfile.get()),
username.orElse(null),
null,
profileBadgeConverter.convert(acceptableLanguages, accountProfile.get().getBadges(), isSelf),
credential.orElse(null)));
} catch (InvalidInputException e) {
logger.info("Bad profile request", e);
throw new WebApplicationException(Response.Status.BAD_REQUEST);
}
}
@@ -342,58 +308,58 @@ public class ProfileController {
username = username.toLowerCase();
final Account accountProfile = accountsManager.getByUsername(username)
.orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build()));
Optional<UUID> uuid = usernamesManager.get(username);
final boolean isSelf = auth.getAccount().getUuid().equals(accountProfile.getUuid());
if (uuid.isEmpty()) {
throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build());
}
final boolean isSelf = auth.getAccount().getUuid().equals(uuid.get());
Optional<Account> accountProfile = accountsManager.get(uuid.get());
if (accountProfile.isEmpty()) {
throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build());
}
return new Profile(
accountProfile.getProfileName(),
accountProfile.get().getProfileName(),
null,
null,
accountProfile.getAvatar(),
accountProfile.get().getAvatar(),
null,
accountProfile.getIdentityKey(),
UnidentifiedAccessChecksum.generateFor(accountProfile.getUnidentifiedAccessKey()),
accountProfile.isUnrestrictedUnidentifiedAccess(),
UserCapabilities.createForAccount(accountProfile),
accountProfile.get().getIdentityKey(),
UnidentifiedAccessChecksum.generateFor(accountProfile.get().getUnidentifiedAccessKey()),
accountProfile.get().isUnrestrictedUnidentifiedAccess(),
UserCapabilities.createForAccount(accountProfile.get()),
username,
accountProfile.getUuid(),
accountProfile.get().getUuid(),
profileBadgeConverter.convert(
getAcceptableLanguagesForRequest(containerRequestContext),
accountProfile.getBadges(),
accountProfile.get().getBadges(),
isSelf),
null,
null);
}
private ProfileKeyCredentialResponse getProfileCredential(final String encodedProfileCredentialRequest,
final VersionedProfile profile,
final UUID uuid) {
try {
final ProfileKeyCommitment commitment = new ProfileKeyCommitment(profile.getCommitment());
final ProfileKeyCredentialRequest request = new ProfileKeyCredentialRequest(Hex.decodeHex(encodedProfileCredentialRequest));
private Optional<ProfileKeyCredentialResponse> getProfileCredential(Optional<String> encodedProfileCredentialRequest,
Optional<VersionedProfile> profile,
UUID uuid)
throws InvalidInputException
{
if (encodedProfileCredentialRequest.isEmpty()) return Optional.empty();
if (profile.isEmpty()) return Optional.empty();
return zkProfileOperations.issueProfileKeyCredential(request, uuid, commitment);
} catch (DecoderException | VerificationFailedException | InvalidInputException e) {
try {
ProfileKeyCommitment commitment = new ProfileKeyCommitment(profile.get().getCommitment());
ProfileKeyCredentialRequest request = new ProfileKeyCredentialRequest(Hex.decodeHex(encodedProfileCredentialRequest.get()));
ProfileKeyCredentialResponse response = zkProfileOperations.issueProfileKeyCredential(request, uuid, commitment);
return Optional.of(response);
} catch (DecoderException | VerificationFailedException e) {
throw new WebApplicationException(e, Response.status(Response.Status.BAD_REQUEST).build());
}
}
private PniCredentialResponse getPniCredential(final String encodedCredentialRequest,
final VersionedProfile profile,
final UUID accountIdentifier,
final UUID phoneNumberIdentifier) {
try {
final ProfileKeyCommitment commitment = new ProfileKeyCommitment(profile.getCommitment());
final ProfileKeyCredentialRequest request = new ProfileKeyCredentialRequest(Hex.decodeHex(encodedCredentialRequest));
return zkProfileOperations.issuePniCredential(request, accountIdentifier, phoneNumberIdentifier, commitment);
} catch (DecoderException | VerificationFailedException | InvalidInputException e) {
throw new WebApplicationException(e, Response.status(Response.Status.BAD_REQUEST).build());
}
}
// Old profile endpoints. Replaced by versioned profile endpoints (above)
@@ -402,7 +368,7 @@ public class ProfileController {
@PUT
@Produces(MediaType.APPLICATION_JSON)
@Path("/name/{name}")
public void setLegacyProfile(@Auth AuthenticatedAccount auth,
public void setProfile(@Auth AuthenticatedAccount auth,
@PathParam("name") @ExactlySize(value = {72, 108}, payload = {Unwrapping.Unwrap.class}) Optional<String> name) {
accountsManager.update(auth.getAccount(), a -> a.setProfileName(name.orElse(null)));
}
@@ -412,7 +378,7 @@ public class ProfileController {
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/{identifier}")
public Profile getLegacyProfile(
public Profile getProfile(
@Auth Optional<AuthenticatedAccount> auth,
@HeaderParam(OptionalAccess.UNIDENTIFIED) Optional<Anonymous> accessKey,
@Context ContainerRequestContext containerRequestContext,
@@ -425,8 +391,6 @@ public class ProfileController {
throw new WebApplicationException(Response.Status.UNAUTHORIZED);
}
Metrics.counter(LEGACY_GET_PROFILE_COUNTER_NAME, Tags.of(UserAgentTagUtil.getPlatformTag(userAgent))).increment();
boolean isSelf = false;
if (auth.isPresent()) {
UUID authedUuid = auth.get().getAccount().getUuid();
@@ -434,10 +398,10 @@ public class ProfileController {
isSelf = authedUuid.equals(identifier);
}
Optional<Account> accountProfile = accountsManager.getByAccountIdentifier(identifier);
Optional<Account> accountProfile = accountsManager.get(identifier);
OptionalAccess.verify(auth.map(AuthenticatedAccount::getAccount), accessKey, accountProfile);
Optional<String> username = accountProfile.flatMap(Account::getUsername);
Optional<String> username = usernamesManager.get(accountProfile.get().getUuid());
return new Profile(
accountProfile.get().getProfileName(),
@@ -455,7 +419,6 @@ public class ProfileController {
getAcceptableLanguagesForRequest(containerRequestContext),
accountProfile.get().getBadges(),
isSelf),
null,
null);
}
@@ -464,7 +427,7 @@ public class ProfileController {
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/form/avatar")
public ProfileAvatarUploadAttributes getLegacyAvatarUploadForm(@Auth AuthenticatedAccount auth) {
public ProfileAvatarUploadAttributes getAvatarUploadForm(@Auth AuthenticatedAccount auth) {
String previousAvatar = auth.getAccount().getAvatar();
String objectName = generateAvatarObjectName();
ProfileAvatarUploadAttributes profileAvatarUploadAttributes = generateAvatarUploadForm(objectName);
@@ -21,6 +21,7 @@ import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Base64;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -54,7 +55,6 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.apache.commons.lang3.StringUtils;
import org.signal.zkgroup.InvalidInputException;
import org.signal.zkgroup.VerificationFailedException;
import org.signal.zkgroup.receipts.ReceiptCredentialRequest;
@@ -562,13 +562,10 @@ public class SubscriptionController {
return stripeManager.getPaymentIntent(request.getPaymentIntentId())
.thenCompose(paymentIntent -> {
if (paymentIntent == null) {
throw new WebApplicationException(Status.NOT_FOUND);
}
if (StringUtils.equalsIgnoreCase("processing", paymentIntent.getStatus())) {
throw new WebApplicationException(Status.NO_CONTENT);
}
if (!StringUtils.equalsIgnoreCase("succeeded", paymentIntent.getStatus())) {
throw new WebApplicationException(Status.PAYMENT_REQUIRED);
if (!"succeeded".equalsIgnoreCase(paymentIntent.getStatus())) {
throw new WebApplicationException(Status.NO_CONTENT);
}
ReceiptCredentialRequest receiptCredentialRequest;
try {
@@ -606,7 +603,6 @@ public class SubscriptionController {
private final boolean cancelAtPeriodEnd;
private final String currency;
private final BigDecimal amount;
private final String status;
public Subscription(
@JsonProperty("level") long level,
@@ -615,8 +611,7 @@ public class SubscriptionController {
@JsonProperty("active") boolean active,
@JsonProperty("cancelAtPeriodEnd") boolean cancelAtPeriodEnd,
@JsonProperty("currency") String currency,
@JsonProperty("amount") BigDecimal amount,
@JsonProperty("status") String status) {
@JsonProperty("amount") BigDecimal amount) {
this.level = level;
this.billingCycleAnchor = billingCycleAnchor;
this.endOfCurrentPeriod = endOfCurrentPeriod;
@@ -624,7 +619,6 @@ public class SubscriptionController {
this.cancelAtPeriodEnd = cancelAtPeriodEnd;
this.currency = currency;
this.amount = amount;
this.status = status;
}
public long getLevel() {
@@ -654,10 +648,6 @@ public class SubscriptionController {
public BigDecimal getAmount() {
return amount;
}
public String getStatus() {
return status;
}
}
private final Subscription subscription;
@@ -698,8 +688,7 @@ public class SubscriptionController {
Objects.equals(subscription.getStatus(), "active"),
subscription.getCancelAtPeriodEnd(),
price.getCurrency().toUpperCase(Locale.ROOT),
price.getUnitAmountDecimal(),
subscription.getStatus()
price.getUnitAmountDecimal()
))).build())));
});
}
@@ -741,7 +730,7 @@ public class SubscriptionController {
@Path("/{subscriberId}/receipt_credentials")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public CompletableFuture<Response> createSubscriptionReceiptCredentials(
public CompletableFuture<Response> getReceiptCredentials(
@Auth Optional<AuthenticatedAccount> authenticatedAccount,
@PathParam("subscriberId") String subscriberId,
@Valid GetReceiptCredentialsRequest request) {
@@ -750,7 +739,7 @@ public class SubscriptionController {
.thenApply(this::requireRecordFromGetResult)
.thenCompose(record -> {
if (record.subscriptionId == null) {
return CompletableFuture.completedFuture(Response.status(Status.NOT_FOUND).build());
return CompletableFuture.completedFuture(Response.noContent().build());
}
ReceiptCredentialRequest receiptCredentialRequest;
try {
@@ -758,20 +747,29 @@ public class SubscriptionController {
} catch (InvalidInputException e) {
throw new BadRequestException("invalid receipt credential request", e);
}
return stripeManager.getLatestInvoiceForSubscription(record.subscriptionId)
.thenCompose(invoice -> convertInvoiceToReceipt(invoice, record.subscriptionId))
.thenCompose(receipt -> issuedReceiptsManager.recordIssuance(
receipt.getInvoiceLineItemId(), receiptCredentialRequest, requestData.now)
.thenApply(unused -> receipt))
return stripeManager.getPaidInvoicesForSubscription(record.subscriptionId, requestData.now)
.thenCompose(invoices -> checkNextInvoice(invoices.iterator(), record.subscriptionId))
.thenCompose(receipt -> {
if (receipt == null) {
return CompletableFuture.completedFuture(null);
}
return issuedReceiptsManager.recordIssuance(
receipt.getInvoiceLineItemId(), receiptCredentialRequest, requestData.now)
.thenApply(unused -> receipt);
})
.thenApply(receipt -> {
ReceiptCredentialResponse receiptCredentialResponse;
try {
receiptCredentialResponse = zkReceiptOperations.issueReceiptCredential(
receiptCredentialRequest, receipt.getExpiration().getEpochSecond(), receipt.getLevel());
} catch (VerificationFailedException e) {
throw new BadRequestException("receipt credential request failed verification", e);
if (receipt == null) {
return Response.noContent().build();
} else {
ReceiptCredentialResponse receiptCredentialResponse;
try {
receiptCredentialResponse = zkReceiptOperations.issueReceiptCredential(
receiptCredentialRequest, receipt.getExpiration().getEpochSecond(), receipt.getLevel());
} catch (VerificationFailedException e) {
throw new BadRequestException("receipt credential request failed verification", e);
}
return Response.ok(new GetReceiptCredentialsResponse(receiptCredentialResponse.serialize())).build();
}
return Response.ok(new GetReceiptCredentialsResponse(receiptCredentialResponse.serialize())).build();
});
});
}
@@ -801,46 +799,35 @@ public class SubscriptionController {
}
}
private CompletableFuture<Receipt> convertInvoiceToReceipt(Invoice latestSubscriptionInvoice, String subscriptionId) {
if (latestSubscriptionInvoice == null) {
throw new WebApplicationException(Status.NO_CONTENT);
}
if (StringUtils.equalsIgnoreCase("open", latestSubscriptionInvoice.getStatus())) {
throw new WebApplicationException(Status.NO_CONTENT);
}
if (!StringUtils.equalsIgnoreCase("paid", latestSubscriptionInvoice.getStatus())) {
throw new WebApplicationException(Status.PAYMENT_REQUIRED);
private CompletableFuture<Receipt> checkNextInvoice(Iterator<Invoice> invoiceIterator, String subscriptionId) {
if (!invoiceIterator.hasNext()) {
return null;
}
return stripeManager.getInvoiceLineItemsForInvoice(latestSubscriptionInvoice).thenCompose(invoiceLineItems -> {
Invoice invoice = invoiceIterator.next();
return stripeManager.getInvoiceLineItemsForInvoice(invoice).thenCompose(invoiceLineItems -> {
Collection<InvoiceLineItem> subscriptionLineItems = invoiceLineItems.stream()
.filter(invoiceLineItem -> Objects.equals("subscription", invoiceLineItem.getType()))
.collect(Collectors.toList());
if (subscriptionLineItems.isEmpty()) {
throw new IllegalStateException("latest subscription invoice has no subscription line items; subscriptionId="
+ subscriptionId + "; invoiceId=" + latestSubscriptionInvoice.getId());
return checkNextInvoice(invoiceIterator, subscriptionId);
}
if (subscriptionLineItems.size() > 1) {
throw new IllegalStateException(
"latest subscription invoice has too many subscription line items; subscriptionId=" + subscriptionId
+ "; invoiceId=" + latestSubscriptionInvoice.getId() + "; count=" + subscriptionLineItems.size());
throw new IllegalStateException("invoice has more than one subscription; subscriptionId=" + subscriptionId
+ "; count=" + subscriptionLineItems.size());
}
InvoiceLineItem subscriptionLineItem = subscriptionLineItems.stream().findAny().get();
return getReceiptForSubscriptionInvoiceLineItem(subscriptionLineItem);
return stripeManager.getProductForPrice(subscriptionLineItem.getPrice().getId()).thenApply(product -> new Receipt(
Instant.ofEpochSecond(subscriptionLineItem.getPeriod().getEnd())
.plus(subscriptionConfiguration.getBadgeGracePeriod())
.truncatedTo(ChronoUnit.DAYS)
.plus(1, ChronoUnit.DAYS),
stripeManager.getLevelForProduct(product),
subscriptionLineItem.getId()));
});
}
private CompletableFuture<Receipt> getReceiptForSubscriptionInvoiceLineItem(InvoiceLineItem subscriptionLineItem) {
return stripeManager.getProductForPrice(subscriptionLineItem.getPrice().getId()).thenApply(product -> new Receipt(
Instant.ofEpochSecond(subscriptionLineItem.getPeriod().getEnd())
.plus(subscriptionConfiguration.getBadgeGracePeriod())
.truncatedTo(ChronoUnit.DAYS)
.plus(1, ChronoUnit.DAYS),
stripeManager.getLevelForProduct(product),
subscriptionLineItem.getId()));
}
private SubscriptionManager.Record requireRecordFromGetResult(SubscriptionManager.GetResult getResult) {
if (getResult == GetResult.PASSWORD_MISMATCH) {
throw new ForbiddenException("subscriberId mismatch");
@@ -23,34 +23,32 @@ import org.whispersystems.textsecuregcm.util.Util;
@Path("/v1/voice/")
public class VoiceVerificationController {
private static final String PLAY_TWIML = """
<?xml version="1.0" encoding="UTF-8"?>
<Response>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Pause length="1"/>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Pause length="1"/>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
<Play>%s</Play>
</Response>
""";
private static final String PLAY_TWIML = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
"<Response>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Pause length=\"1\"/>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Pause length=\"1\"/>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
" <Play>%s</Play>\n" +
"</Response>";
private static final String DEFAULT_LOCALE = "en-US";
@@ -8,7 +8,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import javax.validation.constraints.Size;
import org.whispersystems.textsecuregcm.storage.Device.DeviceCapabilities;
import org.whispersystems.textsecuregcm.util.ExactlySize;
public class AccountAttributes {
@@ -26,7 +25,6 @@ public class AccountAttributes {
private String registrationLock;
@JsonProperty
@ExactlySize({0, 16})
private byte[] unidentifiedAccessKey;
@JsonProperty
@@ -82,9 +80,4 @@ public class AccountAttributes {
public boolean isDiscoverableByPhoneNumber() {
return discoverableByPhoneNumber;
}
@VisibleForTesting
public void setUnidentifiedAccessKey(final byte[] unidentifiedAccessKey) {
this.unidentifiedAccessKey = unidentifiedAccessKey;
}
}
@@ -8,25 +8,29 @@ package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.whispersystems.textsecuregcm.auth.ExternalServiceCredentials;
import java.util.UUID;
public class AccountCreationResult {
@JsonProperty
private final UUID uuid;
@JsonProperty
private final String number;
private final UUID pni;
@JsonProperty
private final boolean storageCapable;
@JsonCreator
public AccountCreationResult(
@JsonProperty("uuid") final UUID uuid,
@JsonProperty("number") final String number,
@JsonProperty("pni") final UUID pni,
@JsonProperty("storageCapable") final boolean storageCapable) {
this.uuid = uuid;
this.number = number;
this.pni = pni;
this.storageCapable = storageCapable;
}
@@ -38,10 +42,6 @@ public class AccountCreationResult {
return number;
}
public UUID getPni() {
return pni;
}
public boolean isStorageCapable() {
return storageCapable;
}
@@ -72,15 +72,6 @@ public class Badge {
return svgs;
}
/**
* Workaround for old Android builds that expect this field to exist but don't care it's an empty string.
*/
@Deprecated
@JsonProperty
public String getImageUrl() {
return "";
}
@Override
public boolean equals(final Object o) {
if (this == o) {
@@ -14,11 +14,13 @@ import javax.validation.constraints.NotEmpty;
public class BadgeSvg {
private final String light;
private final String dark;
private final String transparent;
@JsonCreator
public BadgeSvg(
@JsonProperty("light") final String light,
@JsonProperty("dark") final String dark) {
@JsonProperty("dark") final String dark,
@JsonProperty("transparent") final String transparent) {
if (Strings.isNullOrEmpty(light)) {
throw new IllegalArgumentException("light cannot be empty");
}
@@ -27,6 +29,10 @@ public class BadgeSvg {
throw new IllegalArgumentException("dark cannot be empty");
}
this.dark = dark;
if (Strings.isNullOrEmpty(transparent)) {
throw new IllegalArgumentException("transparent cannot be empty");
}
this.transparent = transparent;
}
@NotEmpty
@@ -39,6 +45,11 @@ public class BadgeSvg {
return dark;
}
@NotEmpty
public String getTransparent() {
return transparent;
}
@Override
public boolean equals(final Object o) {
if (this == o) {
@@ -49,12 +60,13 @@ public class BadgeSvg {
}
BadgeSvg badgeSvg = (BadgeSvg) o;
return Objects.equals(light, badgeSvg.light)
&& Objects.equals(dark, badgeSvg.dark);
&& Objects.equals(dark, badgeSvg.dark)
&& Objects.equals(transparent, badgeSvg.transparent);
}
@Override
public int hashCode() {
return Objects.hash(light, dark);
return Objects.hash(light, dark, transparent);
}
@Override
@@ -62,6 +74,7 @@ public class BadgeSvg {
return "BadgeSvg{" +
"light='" + light + '\'' +
", dark='" + dark + '\'' +
", transparent='" + transparent + '\'' +
'}';
}
}
@@ -8,9 +8,8 @@ package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import javax.annotation.Nullable;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import org.apache.commons.lang3.StringUtils;
@@ -43,8 +42,8 @@ public class CreateProfileRequest {
private String paymentAddress;
@JsonProperty
@Nullable
private List<String> badgeIds;
@NotNull
private List<String> badgeIds = new ArrayList<>();
@JsonProperty
@NotNull
@@ -96,7 +95,7 @@ public class CreateProfileRequest {
return StringUtils.stripToNull(paymentAddress);
}
public Optional<List<String>> getBadges() {
return Optional.ofNullable(badgeIds);
public List<String> getBadges() {
return badgeIds;
}
}
@@ -8,14 +8,7 @@ package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import java.util.UUID;
public class DeviceResponse {
@JsonProperty
private UUID uuid;
@JsonProperty
private UUID pni;
@JsonProperty
private long deviceId;
@@ -23,20 +16,10 @@ public class DeviceResponse {
@VisibleForTesting
public DeviceResponse() {}
public DeviceResponse(UUID uuid, UUID pni, long deviceId) {
this.uuid = uuid;
this.pni = pni;
public DeviceResponse(long deviceId) {
this.deviceId = deviceId;
}
public UUID getUuid() {
return uuid;
}
public UUID getPni() {
return pni;
}
public long getDeviceId() {
return deviceId;
}
@@ -5,39 +5,60 @@
package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Arrays;
import java.util.Objects;
import java.util.UUID;
public class OutgoingMessageEntity {
private final UUID guid;
private final int type;
private final String relay;
private final long timestamp;
private final String source;
private final UUID sourceUuid;
private final int sourceDevice;
private final UUID destinationUuid;
private final byte[] message;
private final byte[] content;
private final long serverTimestamp;
@JsonIgnore
private long id;
@JsonCreator
public OutgoingMessageEntity(@JsonProperty("guid") final UUID guid,
@JsonProperty("type") final int type,
@JsonProperty("relay") final String relay,
@JsonProperty("timestamp") final long timestamp,
@JsonProperty("source") final String source,
@JsonProperty("sourceUuid") final UUID sourceUuid,
@JsonProperty("sourceDevice") final int sourceDevice,
@JsonProperty("destinationUuid") final UUID destinationUuid,
@JsonProperty("message") final byte[] message,
@JsonProperty("content") final byte[] content,
@JsonProperty("serverTimestamp") final long serverTimestamp)
@JsonIgnore
private boolean cached;
@JsonProperty
private UUID guid;
@JsonProperty
private int type;
@JsonProperty
private String relay;
@JsonProperty
private long timestamp;
@JsonProperty
private String source;
@JsonProperty
private UUID sourceUuid;
@JsonProperty
private int sourceDevice;
@JsonProperty
private byte[] message;
@JsonProperty
private byte[] content;
@JsonProperty
private long serverTimestamp;
public OutgoingMessageEntity() {}
public OutgoingMessageEntity(long id, boolean cached,
UUID guid, int type, String relay, long timestamp,
String source, UUID sourceUuid, int sourceDevice,
byte[] message, byte[] content, long serverTimestamp)
{
this.id = id;
this.cached = cached;
this.guid = guid;
this.type = type;
this.relay = relay;
@@ -45,7 +66,6 @@ public class OutgoingMessageEntity {
this.source = source;
this.sourceUuid = sourceUuid;
this.sourceDevice = sourceDevice;
this.destinationUuid = destinationUuid;
this.message = message;
this.content = content;
this.serverTimestamp = serverTimestamp;
@@ -79,10 +99,6 @@ public class OutgoingMessageEntity {
return sourceDevice;
}
public UUID getDestinationUuid() {
return destinationUuid;
}
public byte[] getMessage() {
return message;
}
@@ -91,6 +107,16 @@ public class OutgoingMessageEntity {
return content;
}
@JsonIgnore
public long getId() {
return id;
}
@JsonIgnore
public boolean isCached() {
return cached;
}
public long getServerTimestamp() {
return serverTimestamp;
}
@@ -100,22 +126,23 @@ public class OutgoingMessageEntity {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
final OutgoingMessageEntity that = (OutgoingMessageEntity)o;
return type == that.type &&
return id == that.id &&
cached == that.cached &&
type == that.type &&
timestamp == that.timestamp &&
sourceDevice == that.sourceDevice &&
serverTimestamp == that.serverTimestamp &&
guid.equals(that.guid) &&
Objects.equals(guid, that.guid) &&
Objects.equals(relay, that.relay) &&
Objects.equals(source, that.source) &&
Objects.equals(sourceUuid, that.sourceUuid) &&
destinationUuid.equals(that.destinationUuid) &&
Arrays.equals(message, that.message) &&
Arrays.equals(content, that.content);
}
@Override
public int hashCode() {
int result = Objects.hash(guid, type, relay, timestamp, source, sourceUuid, sourceDevice, destinationUuid, serverTimestamp);
int result = Objects.hash(id, cached, guid, type, relay, timestamp, source, sourceUuid, sourceDevice, serverTimestamp);
result = 31 * result + Arrays.hashCode(message);
result = 31 * result + Arrays.hashCode(content);
return result;
@@ -1,41 +0,0 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.entities;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import java.io.IOException;
import java.util.Base64;
import org.signal.zkgroup.InvalidInputException;
import org.signal.zkgroup.profiles.PniCredentialResponse;
public class PniCredentialResponseAdapter {
public static class Serializing extends JsonSerializer<PniCredentialResponse> {
@Override
public void serialize(PniCredentialResponse response, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
if (response == null) jsonGenerator.writeNull();
else jsonGenerator.writeString(Base64.getEncoder().encodeToString(response.serialize()));
}
}
public static class Deserializing extends JsonDeserializer<PniCredentialResponse> {
@Override
public PniCredentialResponse deserialize(JsonParser jsonParser, DeserializationContext deserializationContext)
throws IOException {
try {
return new PniCredentialResponse(Base64.getDecoder().decode(jsonParser.getValueAsString()));
} catch (InvalidInputException e) {
throw new IOException(e);
}
}
}
}
@@ -11,7 +11,6 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.google.common.annotations.VisibleForTesting;
import java.util.List;
import java.util.UUID;
import org.signal.zkgroup.profiles.PniCredentialResponse;
import org.signal.zkgroup.profiles.ProfileKeyCredentialResponse;
public class Profile {
@@ -57,17 +56,12 @@ public class Profile {
@JsonDeserialize(using = ProfileKeyCredentialResponseAdapter.Deserializing.class)
private ProfileKeyCredentialResponse credential;
@JsonProperty
@JsonSerialize(using = PniCredentialResponseAdapter.Serializing.class)
@JsonDeserialize(using = PniCredentialResponseAdapter.Deserializing.class)
private PniCredentialResponse pniCredential;
public Profile() {}
public Profile(
String name, String about, String aboutEmoji, String avatar, String paymentAddress, String identityKey,
String unidentifiedAccess, boolean unrestrictedUnidentifiedAccess, UserCapabilities capabilities, String username,
UUID uuid, List<Badge> badges, ProfileKeyCredentialResponse credential, PniCredentialResponse pniCredential)
UUID uuid, List<Badge> badges, ProfileKeyCredentialResponse credential)
{
this.name = name;
this.about = about;
@@ -82,7 +76,6 @@ public class Profile {
this.uuid = uuid;
this.badges = badges;
this.credential = credential;
this.pniCredential = pniCredential;
}
@VisibleForTesting
@@ -140,12 +133,4 @@ public class Profile {
public List<Badge> getBadges() {
return badges;
}
public ProfileKeyCredentialResponse getCredential() {
return credential;
}
public PniCredentialResponse getPniCredential() {
return pniCredential;
}
}
@@ -1,130 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.limits;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import org.whispersystems.textsecuregcm.configuration.RateLimitsConfiguration.RateLimitConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
public class DynamicRateLimiters {
private final FaultTolerantRedisCluster cacheCluster;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
private final AtomicReference<RateLimiter> rateLimitResetLimiter;
private final AtomicReference<RateLimiter> recaptchaChallengeAttemptLimiter;
private final AtomicReference<RateLimiter> recaptchaChallengeSuccessLimiter;
private final AtomicReference<RateLimiter> pushChallengeAttemptLimiter;
private final AtomicReference<RateLimiter> pushChallengeSuccessLimiter;
public DynamicRateLimiters(final FaultTolerantRedisCluster rateLimitCluster,
final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) {
this.cacheCluster = rateLimitCluster;
this.dynamicConfigurationManager = dynamicConfigurationManager;
this.rateLimitResetLimiter = new AtomicReference<>(
createRateLimitResetLimiter(this.cacheCluster,
this.dynamicConfigurationManager.getConfiguration().getLimits().getRateLimitReset()));
this.recaptchaChallengeAttemptLimiter = new AtomicReference<>(createRecaptchaChallengeAttemptLimiter(
this.cacheCluster,
this.dynamicConfigurationManager.getConfiguration().getLimits().getRecaptchaChallengeAttempt()));
this.recaptchaChallengeSuccessLimiter = new AtomicReference<>(createRecaptchaChallengeSuccessLimiter(
this.cacheCluster,
this.dynamicConfigurationManager.getConfiguration().getLimits().getRecaptchaChallengeSuccess()));
this.pushChallengeAttemptLimiter = new AtomicReference<>(createPushChallengeAttemptLimiter(this.cacheCluster,
this.dynamicConfigurationManager.getConfiguration().getLimits().getPushChallengeAttempt()));
this.pushChallengeSuccessLimiter = new AtomicReference<>(createPushChallengeSuccessLimiter(this.cacheCluster,
this.dynamicConfigurationManager.getConfiguration().getLimits().getPushChallengeSuccess()));
}
public RateLimiter getRateLimitResetLimiter() {
return updateAndGetRateLimiter(
rateLimitResetLimiter,
dynamicConfigurationManager.getConfiguration().getLimits().getRateLimitReset(),
this::createRateLimitResetLimiter);
}
public RateLimiter getRecaptchaChallengeAttemptLimiter() {
return updateAndGetRateLimiter(
recaptchaChallengeAttemptLimiter,
dynamicConfigurationManager.getConfiguration().getLimits().getRecaptchaChallengeAttempt(),
this::createRecaptchaChallengeAttemptLimiter);
}
public RateLimiter getRecaptchaChallengeSuccessLimiter() {
return updateAndGetRateLimiter(
recaptchaChallengeSuccessLimiter,
dynamicConfigurationManager.getConfiguration().getLimits().getRecaptchaChallengeSuccess(),
this::createRecaptchaChallengeSuccessLimiter);
}
public RateLimiter getPushChallengeAttemptLimiter() {
return updateAndGetRateLimiter(
pushChallengeAttemptLimiter,
dynamicConfigurationManager.getConfiguration().getLimits().getPushChallengeAttempt(),
this::createPushChallengeAttemptLimiter);
}
public RateLimiter getPushChallengeSuccessLimiter() {
return updateAndGetRateLimiter(
pushChallengeSuccessLimiter,
dynamicConfigurationManager.getConfiguration().getLimits().getPushChallengeSuccess(),
this::createPushChallengeSuccessLimiter);
}
private RateLimiter updateAndGetRateLimiter(final AtomicReference<RateLimiter> rateLimiter,
RateLimitConfiguration currentConfiguration,
BiFunction<FaultTolerantRedisCluster, RateLimitConfiguration, RateLimiter> rateLimitFactory) {
return rateLimiter.updateAndGet(limiter -> {
if (limiter.hasConfiguration(currentConfiguration)) {
return limiter;
} else {
return rateLimitFactory.apply(cacheCluster, currentConfiguration);
}
});
}
public RateLimiter createRateLimitResetLimiter(FaultTolerantRedisCluster cacheCluster,
RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "rateLimitReset");
}
public RateLimiter createRecaptchaChallengeAttemptLimiter(FaultTolerantRedisCluster cacheCluster,
RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "recaptchaChallengeAttempt");
}
public RateLimiter createRecaptchaChallengeSuccessLimiter(FaultTolerantRedisCluster cacheCluster,
RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "recaptchaChallengeSuccess");
}
public RateLimiter createPushChallengeAttemptLimiter(FaultTolerantRedisCluster cacheCluster,
RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "pushChallengeAttempt");
}
public RateLimiter createPushChallengeSuccessLimiter(FaultTolerantRedisCluster cacheCluster,
RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "pushChallengeSuccess");
}
private RateLimiter createLimiter(FaultTolerantRedisCluster cacheCluster, RateLimitConfiguration configuration,
String name) {
return new RateLimiter(cacheCluster, name,
configuration.getBucketSize(),
configuration.getLeakRatePerMinute());
}
}
@@ -0,0 +1,79 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.limits;
import static com.codahale.metrics.MetricRegistry.name;
import io.dropwizard.util.Duration;
import io.micrometer.core.instrument.Metrics;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.util.Util;
public class PreKeyRateLimiter {
private static final String RATE_LIMIT_RESET_COUNTER_NAME = name(PreKeyRateLimiter.class, "reset");
private static final String RATE_LIMITED_PREKEYS_COUNTER_NAME = name(PreKeyRateLimiter.class, "rateLimited");
private static final String RATE_LIMITED_PREKEYS_TOTAL_ACCOUNTS_COUNTER_NAME = name(PreKeyRateLimiter.class, "rateLimitedTotal");
private static final String RATE_LIMITED_PREKEYS_ACCOUNTS_ENFORCED_COUNTER_NAME = name(PreKeyRateLimiter.class, "rateLimitedAccountsEnforced");
private static final String RATE_LIMITED_PREKEYS_ACCOUNTS_UNENFORCED_COUNTER_NAME = name(PreKeyRateLimiter.class, "rateLimitedAccountsUnenforced");
private static final String RATE_LIMITED_ACCOUNTS_HLL_KEY = "PreKeyRateLimiter::rateLimitedAccounts";
private static final String RATE_LIMITED_ACCOUNTS_ENFORCED_HLL_KEY = "PreKeyRateLimiter::rateLimitedAccounts::enforced";
private static final String RATE_LIMITED_ACCOUNTS_UNENFORCED_HLL_KEY = "PreKeyRateLimiter::rateLimitedAccounts::unenforced";
private static final long RATE_LIMITED_ACCOUNTS_HLL_TTL_SECONDS = Duration.days(1).toSeconds();
private final RateLimiters rateLimiters;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
private final RateLimitResetMetricsManager metricsManager;
public PreKeyRateLimiter(final RateLimiters rateLimiters,
final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
final RateLimitResetMetricsManager metricsManager) {
this.rateLimiters = rateLimiters;
this.dynamicConfigurationManager = dynamicConfigurationManager;
this.metricsManager = metricsManager;
metricsManager.initializeFunctionCounters(RATE_LIMITED_PREKEYS_TOTAL_ACCOUNTS_COUNTER_NAME,
RATE_LIMITED_ACCOUNTS_HLL_KEY);
metricsManager.initializeFunctionCounters(RATE_LIMITED_PREKEYS_ACCOUNTS_ENFORCED_COUNTER_NAME,
RATE_LIMITED_ACCOUNTS_ENFORCED_HLL_KEY);
metricsManager.initializeFunctionCounters(RATE_LIMITED_PREKEYS_ACCOUNTS_UNENFORCED_COUNTER_NAME,
RATE_LIMITED_ACCOUNTS_UNENFORCED_HLL_KEY);
}
public void validate(final Account account) throws RateLimitExceededException {
try {
rateLimiters.getDailyPreKeysLimiter().validate(account.getUuid());
} catch (final RateLimitExceededException e) {
final boolean enforceLimit = dynamicConfigurationManager.getConfiguration()
.getRateLimitChallengeConfiguration().isPreKeyLimitEnforced();
metricsManager.recordMetrics(account, enforceLimit,
RATE_LIMITED_PREKEYS_COUNTER_NAME,
enforceLimit ? RATE_LIMITED_ACCOUNTS_ENFORCED_HLL_KEY : RATE_LIMITED_ACCOUNTS_UNENFORCED_HLL_KEY,
RATE_LIMITED_ACCOUNTS_HLL_KEY,
RATE_LIMITED_ACCOUNTS_HLL_TTL_SECONDS
);
if (enforceLimit) {
throw e;
}
}
}
public void handleRateLimitReset(final Account account) {
rateLimiters.getDailyPreKeysLimiter().clear(account.getUuid());
Metrics.counter(RATE_LIMIT_RESET_COUNTER_NAME, "countryCode", Util.getCountryCode(account.getNumber()))
.increment();
}
}
@@ -2,26 +2,35 @@ package org.whispersystems.textsecuregcm.limits;
import static com.codahale.metrics.MetricRegistry.name;
import com.vdurmont.semver4j.Semver;
import io.micrometer.core.instrument.Metrics;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.whispersystems.textsecuregcm.abuse.RateLimitChallengeListener;
import java.util.Optional;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
import org.whispersystems.textsecuregcm.push.NotPushRegisteredException;
import org.whispersystems.textsecuregcm.recaptcha.RecaptchaClient;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.util.Util;
import org.whispersystems.textsecuregcm.util.ua.UnrecognizedUserAgentException;
import org.whispersystems.textsecuregcm.util.ua.UserAgent;
import org.whispersystems.textsecuregcm.util.ua.UserAgentUtil;
public class RateLimitChallengeManager {
private final PushChallengeManager pushChallengeManager;
private final RecaptchaClient recaptchaClient;
private final DynamicRateLimiters rateLimiters;
private final PreKeyRateLimiter preKeyRateLimiter;
private final UnsealedSenderRateLimiter unsealedSenderRateLimiter;
private final List<RateLimitChallengeListener> rateLimitChallengeListeners =
Collections.synchronizedList(new ArrayList<>());
private final RateLimiters rateLimiters;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
public static final String OPTION_RECAPTCHA = "recaptcha";
public static final String OPTION_PUSH_CHALLENGE = "pushChallenge";
private static final String RECAPTCHA_ATTEMPT_COUNTER_NAME = name(RateLimitChallengeManager.class, "recaptcha", "attempt");
private static final String RESET_RATE_LIMIT_EXCEEDED_COUNTER_NAME = name(RateLimitChallengeManager.class, "resetRateLimitExceeded");
@@ -32,15 +41,17 @@ public class RateLimitChallengeManager {
public RateLimitChallengeManager(
final PushChallengeManager pushChallengeManager,
final RecaptchaClient recaptchaClient,
final DynamicRateLimiters rateLimiters) {
final PreKeyRateLimiter preKeyRateLimiter,
final UnsealedSenderRateLimiter unsealedSenderRateLimiter,
final RateLimiters rateLimiters,
final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) {
this.pushChallengeManager = pushChallengeManager;
this.recaptchaClient = recaptchaClient;
this.preKeyRateLimiter = preKeyRateLimiter;
this.unsealedSenderRateLimiter = unsealedSenderRateLimiter;
this.rateLimiters = rateLimiters;
}
public void addListener(final RateLimitChallengeListener rateLimitChallengeListener) {
rateLimitChallengeListeners.add(rateLimitChallengeListener);
this.dynamicConfigurationManager = dynamicConfigurationManager;
}
public void answerPushChallenge(final Account account, final String challenge) throws RateLimitExceededException {
@@ -81,7 +92,40 @@ public class RateLimitChallengeManager {
throw e;
}
rateLimitChallengeListeners.forEach(listener -> listener.handleRateLimitChallengeAnswered(account));
preKeyRateLimiter.handleRateLimitReset(account);
unsealedSenderRateLimiter.handleRateLimitReset(account);
}
public boolean isClientBelowMinimumVersion(final String userAgent) {
try {
final UserAgent client = UserAgentUtil.parseUserAgentString(userAgent);
final Optional<Semver> minimumClientVersion = dynamicConfigurationManager.getConfiguration()
.getRateLimitChallengeConfiguration()
.getMinimumSupportedVersion(client.getPlatform());
return minimumClientVersion.map(version -> version.isGreaterThan(client.getVersion()))
.orElse(true);
} catch (final UnrecognizedUserAgentException ignored) {
return false;
}
}
public List<String> getChallengeOptions(final Account account) {
final List<String> options = new ArrayList<>(2);
if (rateLimiters.getRecaptchaChallengeAttemptLimiter().hasAvailablePermits(account.getUuid(), 1) &&
rateLimiters.getRecaptchaChallengeSuccessLimiter().hasAvailablePermits(account.getUuid(), 1)) {
options.add(OPTION_RECAPTCHA);
}
if (rateLimiters.getPushChallengeAttemptLimiter().hasAvailablePermits(account.getUuid(), 1) &&
rateLimiters.getPushChallengeSuccessLimiter().hasAvailablePermits(account.getUuid(), 1)) {
options.add(OPTION_PUSH_CHALLENGE);
}
return options;
}
public void sendPushChallenge(final Account account) throws NotPushRegisteredException {
@@ -1,65 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.limits;
import com.vdurmont.semver4j.Semver;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.util.ua.UnrecognizedUserAgentException;
import org.whispersystems.textsecuregcm.util.ua.UserAgent;
import org.whispersystems.textsecuregcm.util.ua.UserAgentUtil;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class RateLimitChallengeOptionManager {
private final DynamicRateLimiters rateLimiters;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
public static final String OPTION_RECAPTCHA = "recaptcha";
public static final String OPTION_PUSH_CHALLENGE = "pushChallenge";
public RateLimitChallengeOptionManager(final DynamicRateLimiters rateLimiters,
final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) {
this.rateLimiters = rateLimiters;
this.dynamicConfigurationManager = dynamicConfigurationManager;
}
public boolean isClientBelowMinimumVersion(final String userAgent) {
try {
final UserAgent client = UserAgentUtil.parseUserAgentString(userAgent);
final Optional<Semver> minimumClientVersion = dynamicConfigurationManager.getConfiguration()
.getRateLimitChallengeConfiguration()
.getMinimumSupportedVersion(client.getPlatform());
return minimumClientVersion.map(version -> version.isGreaterThan(client.getVersion()))
.orElse(true);
} catch (final UnrecognizedUserAgentException ignored) {
return false;
}
}
public List<String> getChallengeOptions(final Account account) {
final List<String> options = new ArrayList<>(2);
if (rateLimiters.getRecaptchaChallengeAttemptLimiter().hasAvailablePermits(account.getUuid(), 1) &&
rateLimiters.getRecaptchaChallengeSuccessLimiter().hasAvailablePermits(account.getUuid(), 1)) {
options.add(OPTION_RECAPTCHA);
}
if (rateLimiters.getPushChallengeAttemptLimiter().hasAvailablePermits(account.getUuid(), 1) &&
rateLimiters.getPushChallengeSuccessLimiter().hasAvailablePermits(account.getUuid(), 1)) {
options.add(OPTION_PUSH_CHALLENGE);
}
return options;
}
}
@@ -0,0 +1,45 @@
package org.whispersystems.textsecuregcm.limits;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.FunctionCounter;
import io.micrometer.core.instrument.MeterRegistry;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.storage.Account;
public class RateLimitResetMetricsManager {
private final FaultTolerantRedisCluster metricsCluster;
private final MeterRegistry meterRegistry;
public RateLimitResetMetricsManager(
final FaultTolerantRedisCluster metricsCluster, final MeterRegistry meterRegistry) {
this.metricsCluster = metricsCluster;
this.meterRegistry = meterRegistry;
}
void initializeFunctionCounters(String counterKey, String hllKey) {
FunctionCounter
.builder(counterKey, this, manager -> manager.getCount(hllKey))
.register(meterRegistry);
}
Long getCount(final String hllKey) {
return metricsCluster.<Long>withCluster(conn -> conn.sync().pfcount(hllKey));
}
void recordMetrics(Account account, boolean enforced, String counterKey, String hllEnforcedKey, String hllTotalKey,
long hllTtl) {
Counter.builder(counterKey)
.tag("enforced", String.valueOf(enforced))
.register(meterRegistry)
.increment();
metricsCluster.useCluster(connection -> {
connection.sync().pfadd(hllEnforcedKey, account.getUuid().toString());
connection.sync().expire(hllEnforcedKey, hllTtl);
connection.sync().pfadd(hllTotalKey, account.getUuid().toString());
connection.sync().expire(hllTotalKey, hllTtl);
});
}
}
@@ -5,8 +5,14 @@
package org.whispersystems.textsecuregcm.limits;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiFunction;
import org.whispersystems.textsecuregcm.configuration.RateLimitsConfiguration;
import org.whispersystems.textsecuregcm.configuration.RateLimitsConfiguration.CardinalityRateLimitConfiguration;
import org.whispersystems.textsecuregcm.configuration.RateLimitsConfiguration.RateLimitConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
public class RateLimiters {
@@ -33,9 +39,22 @@ public class RateLimiters {
private final RateLimiter usernameLookupLimiter;
private final RateLimiter usernameSetLimiter;
private final RateLimiter checkAccountExistenceLimiter;
private final AtomicReference<CardinalityRateLimiter> unsealedSenderCardinalityLimiter;
private final AtomicReference<RateLimiter> unsealedIpLimiter;
private final AtomicReference<RateLimiter> rateLimitResetLimiter;
private final AtomicReference<RateLimiter> recaptchaChallengeAttemptLimiter;
private final AtomicReference<RateLimiter> recaptchaChallengeSuccessLimiter;
private final AtomicReference<RateLimiter> pushChallengeAttemptLimiter;
private final AtomicReference<RateLimiter> pushChallengeSuccessLimiter;
private final AtomicReference<RateLimiter> dailyPreKeysLimiter;
private final FaultTolerantRedisCluster cacheCluster;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfig;
public RateLimiters(RateLimitsConfiguration config, DynamicConfigurationManager<DynamicConfiguration> dynamicConfig, FaultTolerantRedisCluster cacheCluster) {
this.cacheCluster = cacheCluster;
this.dynamicConfig = dynamicConfig;
public RateLimiters(RateLimitsConfiguration config, FaultTolerantRedisCluster cacheCluster) {
this.smsDestinationLimiter = new RateLimiter(cacheCluster, "smsDestination",
config.getSmsDestination().getBucketSize(),
config.getSmsDestination().getLeakRatePerMinute());
@@ -108,9 +127,92 @@ public class RateLimiters {
config.getUsernameSet().getBucketSize(),
config.getUsernameSet().getLeakRatePerMinute());
this.checkAccountExistenceLimiter = new RateLimiter(cacheCluster, "checkAccountExistence",
config.getCheckAccountExistence().getBucketSize(),
config.getCheckAccountExistence().getLeakRatePerMinute());
this.dailyPreKeysLimiter = new AtomicReference<>(createDailyPreKeysLimiter(cacheCluster, dynamicConfig.getConfiguration().getLimits().getDailyPreKeys()));
this.unsealedSenderCardinalityLimiter = new AtomicReference<>(createUnsealedSenderCardinalityLimiter(cacheCluster, dynamicConfig.getConfiguration().getLimits().getUnsealedSenderNumber()));
this.unsealedIpLimiter = new AtomicReference<>(createUnsealedIpLimiter(cacheCluster, dynamicConfig.getConfiguration().getLimits().getUnsealedSenderIp()));
this.rateLimitResetLimiter = new AtomicReference<>(
createRateLimitResetLimiter(cacheCluster, dynamicConfig.getConfiguration().getLimits().getRateLimitReset()));
this.recaptchaChallengeAttemptLimiter = new AtomicReference<>(createRecaptchaChallengeAttemptLimiter(cacheCluster, dynamicConfig.getConfiguration().getLimits().getRecaptchaChallengeAttempt()));
this.recaptchaChallengeSuccessLimiter = new AtomicReference<>(createRecaptchaChallengeSuccessLimiter(cacheCluster, dynamicConfig.getConfiguration().getLimits().getRecaptchaChallengeSuccess()));
this.pushChallengeAttemptLimiter = new AtomicReference<>(createPushChallengeAttemptLimiter(cacheCluster, dynamicConfig.getConfiguration().getLimits().getPushChallengeAttempt()));
this.pushChallengeSuccessLimiter = new AtomicReference<>(createPushChallengeSuccessLimiter(cacheCluster, dynamicConfig.getConfiguration().getLimits().getPushChallengeSuccess()));
}
public CardinalityRateLimiter getUnsealedSenderCardinalityLimiter() {
CardinalityRateLimitConfiguration currentConfiguration = dynamicConfig.getConfiguration().getLimits().getUnsealedSenderNumber();
return this.unsealedSenderCardinalityLimiter.updateAndGet(rateLimiter -> {
if (rateLimiter.hasConfiguration(currentConfiguration)) {
return rateLimiter;
} else {
return createUnsealedSenderCardinalityLimiter(cacheCluster, currentConfiguration);
}
});
}
public RateLimiter getUnsealedIpLimiter() {
return updateAndGetRateLimiter(
unsealedIpLimiter,
dynamicConfig.getConfiguration().getLimits().getUnsealedSenderIp(),
this::createUnsealedIpLimiter);
}
public RateLimiter getRateLimitResetLimiter() {
return updateAndGetRateLimiter(
rateLimitResetLimiter,
dynamicConfig.getConfiguration().getLimits().getRateLimitReset(),
this::createRateLimitResetLimiter);
}
public RateLimiter getRecaptchaChallengeAttemptLimiter() {
return updateAndGetRateLimiter(
recaptchaChallengeAttemptLimiter,
dynamicConfig.getConfiguration().getLimits().getRecaptchaChallengeAttempt(),
this::createRecaptchaChallengeAttemptLimiter);
}
public RateLimiter getRecaptchaChallengeSuccessLimiter() {
return updateAndGetRateLimiter(
recaptchaChallengeSuccessLimiter,
dynamicConfig.getConfiguration().getLimits().getRecaptchaChallengeSuccess(),
this::createRecaptchaChallengeSuccessLimiter);
}
public RateLimiter getPushChallengeAttemptLimiter() {
return updateAndGetRateLimiter(
pushChallengeAttemptLimiter,
dynamicConfig.getConfiguration().getLimits().getPushChallengeAttempt(),
this::createPushChallengeAttemptLimiter);
}
public RateLimiter getPushChallengeSuccessLimiter() {
return updateAndGetRateLimiter(
pushChallengeSuccessLimiter,
dynamicConfig.getConfiguration().getLimits().getPushChallengeSuccess(),
this::createPushChallengeSuccessLimiter);
}
public RateLimiter getDailyPreKeysLimiter() {
return updateAndGetRateLimiter(
dailyPreKeysLimiter,
dynamicConfig.getConfiguration().getLimits().getDailyPreKeys(),
this::createDailyPreKeysLimiter);
}
private RateLimiter updateAndGetRateLimiter(final AtomicReference<RateLimiter> rateLimiter,
RateLimitConfiguration currentConfiguration,
BiFunction<FaultTolerantRedisCluster, RateLimitConfiguration, RateLimiter> rateLimitFactory) {
return rateLimiter.updateAndGet(limiter -> {
if (limiter.hasConfiguration(currentConfiguration)) {
return limiter;
} else {
return rateLimitFactory.apply(cacheCluster, currentConfiguration);
}
});
}
public RateLimiter getAllocateDeviceLimiter() {
@@ -185,7 +287,42 @@ public class RateLimiters {
return usernameSetLimiter;
}
public RateLimiter getCheckAccountExistenceLimiter() {
return checkAccountExistenceLimiter;
private CardinalityRateLimiter createUnsealedSenderCardinalityLimiter(FaultTolerantRedisCluster cacheCluster, CardinalityRateLimitConfiguration configuration) {
return new CardinalityRateLimiter(cacheCluster, "unsealedSender", configuration.getTtl(), configuration.getMaxCardinality());
}
private RateLimiter createUnsealedIpLimiter(FaultTolerantRedisCluster cacheCluster, RateLimitConfiguration configuration)
{
return createLimiter(cacheCluster, configuration, "unsealedIp");
}
public RateLimiter createRateLimitResetLimiter(FaultTolerantRedisCluster cacheCluster, RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "rateLimitReset");
}
public RateLimiter createRecaptchaChallengeAttemptLimiter(FaultTolerantRedisCluster cacheCluster, RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "recaptchaChallengeAttempt");
}
public RateLimiter createRecaptchaChallengeSuccessLimiter(FaultTolerantRedisCluster cacheCluster, RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "recaptchaChallengeSuccess");
}
public RateLimiter createPushChallengeAttemptLimiter(FaultTolerantRedisCluster cacheCluster, RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "pushChallengeAttempt");
}
public RateLimiter createPushChallengeSuccessLimiter(FaultTolerantRedisCluster cacheCluster, RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "pushChallengeSuccess");
}
public RateLimiter createDailyPreKeysLimiter(FaultTolerantRedisCluster cacheCluster, RateLimitConfiguration configuration) {
return createLimiter(cacheCluster, configuration, "dailyPreKeys");
}
private RateLimiter createLimiter(FaultTolerantRedisCluster cacheCluster, RateLimitConfiguration configuration, String name) {
return new RateLimiter(cacheCluster, name,
configuration.getBucketSize(),
configuration.getLeakRatePerMinute());
}
}
@@ -0,0 +1,115 @@
/*
* Copyright 2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.limits;
import static com.codahale.metrics.MetricRegistry.name;
import io.dropwizard.util.Duration;
import io.lettuce.core.SetArgs;
import io.micrometer.core.instrument.Metrics;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicRateLimitsConfiguration;
import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.storage.Account;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.util.Util;
public class UnsealedSenderRateLimiter {
private final RateLimiters rateLimiters;
private final FaultTolerantRedisCluster rateLimitCluster;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
private final RateLimitResetMetricsManager metricsManager;
private static final String RATE_LIMIT_RESET_COUNTER_NAME = name(UnsealedSenderRateLimiter.class, "reset");
private static final String RATE_LIMITED_UNSEALED_SENDER_COUNTER_NAME = name(UnsealedSenderRateLimiter.class, "rateLimited");
private static final String RATE_LIMITED_UNSEALED_SENDER_ACCOUNTS_TOTAL_COUNTER_NAME = name(UnsealedSenderRateLimiter.class, "rateLimitedAccountsTotal");
private static final String RATE_LIMITED_UNSEALED_SENDER_ACCOUNTS_ENFORCED_COUNTER_NAME = name(UnsealedSenderRateLimiter.class, "rateLimitedAccountsEnforced");
private static final String RATE_LIMITED_UNSEALED_SENDER_ACCOUNTS_UNENFORCED_COUNTER_NAME = name(UnsealedSenderRateLimiter.class, "rateLimitedAccountsUnenforced");
private static final String RATE_LIMITED_ACCOUNTS_HLL_KEY = "UnsealedSenderRateLimiter::rateLimitedAccounts::total";
private static final String RATE_LIMITED_ACCOUNTS_ENFORCED_HLL_KEY = "UnsealedSenderRateLimiter::rateLimitedAccounts::enforced";
private static final String RATE_LIMITED_ACCOUNTS_UNENFORCED_HLL_KEY = "UnsealedSenderRateLimiter::rateLimitedAccounts::unenforced";
private static final long RATE_LIMITED_ACCOUNTS_HLL_TTL_SECONDS = Duration.days(1).toSeconds();
public UnsealedSenderRateLimiter(final RateLimiters rateLimiters,
final FaultTolerantRedisCluster rateLimitCluster,
final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
final RateLimitResetMetricsManager metricsManager) {
this.rateLimiters = rateLimiters;
this.rateLimitCluster = rateLimitCluster;
this.dynamicConfigurationManager = dynamicConfigurationManager;
this.metricsManager = metricsManager;
metricsManager.initializeFunctionCounters(RATE_LIMITED_UNSEALED_SENDER_ACCOUNTS_TOTAL_COUNTER_NAME,
RATE_LIMITED_ACCOUNTS_HLL_KEY);
metricsManager.initializeFunctionCounters(RATE_LIMITED_UNSEALED_SENDER_ACCOUNTS_ENFORCED_COUNTER_NAME,
RATE_LIMITED_ACCOUNTS_ENFORCED_HLL_KEY);
metricsManager.initializeFunctionCounters(RATE_LIMITED_UNSEALED_SENDER_ACCOUNTS_UNENFORCED_COUNTER_NAME,
RATE_LIMITED_ACCOUNTS_UNENFORCED_HLL_KEY);
}
public void validate(final Account sender, final Account destination) throws RateLimitExceededException {
final int maxCardinality = rateLimitCluster.withCluster(connection -> {
final String cardinalityString = connection.sync().get(getMaxCardinalityKey(sender));
return cardinalityString != null
? Integer.parseInt(cardinalityString)
: dynamicConfigurationManager.getConfiguration().getLimits().getUnsealedSenderDefaultCardinalityLimit();
});
try {
rateLimiters.getUnsealedSenderCardinalityLimiter()
.validate(sender.getUuid().toString(), destination.getUuid().toString(), maxCardinality);
} catch (final RateLimitExceededException e) {
final boolean enforceLimit = dynamicConfigurationManager.getConfiguration()
.getRateLimitChallengeConfiguration().isUnsealedSenderLimitEnforced();
metricsManager.recordMetrics(sender, enforceLimit, RATE_LIMITED_UNSEALED_SENDER_COUNTER_NAME,
enforceLimit ? RATE_LIMITED_ACCOUNTS_ENFORCED_HLL_KEY : RATE_LIMITED_ACCOUNTS_UNENFORCED_HLL_KEY,
RATE_LIMITED_ACCOUNTS_HLL_KEY,
RATE_LIMITED_ACCOUNTS_HLL_TTL_SECONDS
);
if (enforceLimit) {
throw e;
}
}
}
public void handleRateLimitReset(final Account account) {
rateLimitCluster.useCluster(connection -> {
final CardinalityRateLimiter unsealedSenderCardinalityLimiter = rateLimiters.getUnsealedSenderCardinalityLimiter();
final DynamicRateLimitsConfiguration rateLimitsConfiguration =
dynamicConfigurationManager.getConfiguration().getLimits();
final long ttl;
{
final long remainingTtl = unsealedSenderCardinalityLimiter.getRemainingTtl(account.getUuid().toString());
ttl = remainingTtl > 0 ? remainingTtl : unsealedSenderCardinalityLimiter.getInitialTtl().toSeconds();
}
final String key = getMaxCardinalityKey(account);
connection.sync().set(key,
String.valueOf(rateLimitsConfiguration.getUnsealedSenderDefaultCardinalityLimit()),
SetArgs.Builder.nx().ex(ttl));
connection.sync().incrby(key, rateLimitsConfiguration.getUnsealedSenderPermitIncrement());
});
Metrics.counter(RATE_LIMIT_RESET_COUNTER_NAME,
"countryCode", Util.getCountryCode(account.getNumber())).increment();
}
private static String getMaxCardinalityKey(final Account account) {
return "max_unsealed_sender_cardinality::" + account.getUuid();
}
}
@@ -1,32 +0,0 @@
/*
* Copyright 2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.mappers;
import java.util.concurrent.CompletionException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
import javax.ws.rs.ext.Provider;
import javax.ws.rs.ext.Providers;
@Provider
public class CompletionExceptionMapper implements ExceptionMapper<CompletionException> {
@Context
private Providers providers;
@Override
public Response toResponse(final CompletionException exception) {
final Throwable cause = exception.getCause();
if (cause != null) {
final Class type = cause.getClass();
return providers.getExceptionMapper(type).toResponse(cause);
}
return Response.serverError().build();
}
}
@@ -10,21 +10,21 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
import org.whispersystems.textsecuregcm.entities.RateLimitChallenge;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeException;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeOptionManager;
import org.whispersystems.textsecuregcm.limits.RateLimitChallengeManager;
public class RateLimitChallengeExceptionMapper implements ExceptionMapper<RateLimitChallengeException> {
private final RateLimitChallengeOptionManager rateLimitChallengeOptionManager;
private final RateLimitChallengeManager rateLimitChallengeManager;
public RateLimitChallengeExceptionMapper(final RateLimitChallengeOptionManager rateLimitChallengeOptionManager) {
this.rateLimitChallengeOptionManager = rateLimitChallengeOptionManager;
public RateLimitChallengeExceptionMapper(final RateLimitChallengeManager rateLimitChallengeManager) {
this.rateLimitChallengeManager = rateLimitChallengeManager;
}
@Override
public Response toResponse(final RateLimitChallengeException exception) {
return Response.status(428)
.entity(new RateLimitChallenge(UUID.randomUUID().toString(),
rateLimitChallengeOptionManager.getChallengeOptions(exception.getAccount())))
rateLimitChallengeManager.getChallengeOptions(exception.getAccount())))
.header("Retry-After", exception.getRetryAfter().toSeconds())
.build();
}
@@ -24,6 +24,6 @@ public class CpuUsageGauge extends CachedGauge<Integer> {
@Override
protected Integer loadValue() {
return (int) Math.ceil(operatingSystemMXBean.getCpuLoad() * 100);
return (int) Math.ceil(operatingSystemMXBean.getSystemCpuLoad() * 100);
}
}
@@ -7,6 +7,7 @@ package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.Gauge;
import com.sun.management.OperatingSystemMXBean;
import java.lang.management.ManagementFactory;
public class FreeMemoryGauge implements Gauge<Long> {
@@ -20,6 +21,6 @@ public class FreeMemoryGauge implements Gauge<Long> {
@Override
public Long getValue() {
return operatingSystemMXBean.getFreeMemorySize();
return operatingSystemMXBean.getFreePhysicalMemorySize();
}
}
@@ -6,34 +6,17 @@
package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.MetricRegistry;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.annotations.VisibleForTesting;
import com.vdurmont.semver4j.Semver;
import io.lettuce.core.SetArgs;
import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import java.time.Clock;
import io.micrometer.core.instrument.Tags;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.ua.UnrecognizedUserAgentException;
import org.whispersystems.textsecuregcm.util.ua.UserAgent;
import org.whispersystems.textsecuregcm.util.ua.UserAgentUtil;
import java.util.concurrent.TimeUnit;
/**
* Measures and records the latency between sending a push notification to a device and that device draining its queue
@@ -45,139 +28,48 @@ import org.whispersystems.textsecuregcm.util.ua.UserAgentUtil;
* device and records the time elapsed since the push notification timestamp as a latency observation.
*/
public class PushLatencyManager {
private static final String TIMER_NAME = MetricRegistry.name(PushLatencyManager.class, "latency");
private static final int TTL = (int)Duration.ofDays(1).toSeconds();
private final FaultTolerantRedisCluster redisCluster;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
private final FaultTolerantRedisCluster redisCluster;
private final Clock clock;
private static final String TIMER_NAME = MetricRegistry.name(PushLatencyManager.class, "latency");
private static final int TTL = (int) Duration.ofDays(1).toSeconds();
private static final Logger log = LoggerFactory.getLogger(PushLatencyManager.class);
@VisibleForTesting
enum PushType {
STANDARD,
VOIP
}
@VisibleForTesting
static class PushRecord {
private final Instant timestamp;
@Nullable
private final PushType pushType;
@JsonCreator
PushRecord(@JsonProperty("timestamp") final Instant timestamp,
@JsonProperty("pushType") @Nullable final PushType pushType) {
this.timestamp = timestamp;
this.pushType = pushType;
public PushLatencyManager(final FaultTolerantRedisCluster redisCluster) {
this.redisCluster = redisCluster;
}
public Instant getTimestamp() {
return timestamp;
public void recordPushSent(final UUID accountUuid, final long deviceId) {
recordPushSent(accountUuid, deviceId, System.currentTimeMillis());
}
@Nullable
public PushType getPushType() {
return pushType;
@VisibleForTesting
void recordPushSent(final UUID accountUuid, final long deviceId, final long currentTime) {
redisCluster.useCluster(connection ->
connection.async().set(getFirstUnacknowledgedPushKey(accountUuid, deviceId), String.valueOf(currentTime), SetArgs.Builder.nx().ex(TTL)));
}
}
public PushLatencyManager(final FaultTolerantRedisCluster redisCluster,
final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) {
this(redisCluster, dynamicConfigurationManager, Clock.systemUTC());
}
@VisibleForTesting
PushLatencyManager(final FaultTolerantRedisCluster redisCluster,
final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager,
final Clock clock) {
this.redisCluster = redisCluster;
this.dynamicConfigurationManager = dynamicConfigurationManager;
this.clock = clock;
}
public void recordPushSent(final UUID accountUuid, final long deviceId, final boolean isVoip) {
try {
final String recordJson = SystemMapper.getMapper().writeValueAsString(
new PushRecord(Instant.now(clock), isVoip ? PushType.VOIP : PushType.STANDARD));
redisCluster.useCluster(connection ->
connection.async().set(getFirstUnacknowledgedPushKey(accountUuid, deviceId),
recordJson,
SetArgs.Builder.nx().ex(TTL)));
} catch (final JsonProcessingException e) {
// This should never happen
log.error("Failed to write push latency record JSON", e);
}
}
public void recordQueueRead(final UUID accountUuid, final long deviceId, final String userAgentString) {
takePushRecord(accountUuid, deviceId).thenAccept(pushRecord -> {
if (pushRecord != null) {
final Duration latency = Duration.between(pushRecord.getTimestamp(), Instant.now());
final List<Tag> tags = new ArrayList<>(2);
tags.add(UserAgentTagUtil.getPlatformTag(userAgentString));
if (pushRecord.getPushType() != null) {
tags.add(Tag.of("pushType", pushRecord.getPushType().name().toLowerCase()));
}
try {
final UserAgent userAgent = UserAgentUtil.parseUserAgentString(userAgentString);
final Set<Semver> instrumentedVersions =
dynamicConfigurationManager.getConfiguration().getPushLatencyConfiguration().getInstrumentedVersions()
.getOrDefault(userAgent.getPlatform(), Collections.emptySet());
if (instrumentedVersions.contains(userAgent.getVersion())) {
tags.add(Tag.of("clientVersion", userAgent.getVersion().toString()));
}
} catch (UnrecognizedUserAgentException ignored) {
}
Metrics.timer(TIMER_NAME, tags).record(latency);
}
});
}
@VisibleForTesting
CompletableFuture<PushRecord> takePushRecord(final UUID accountUuid, final long deviceId) {
final String key = getFirstUnacknowledgedPushKey(accountUuid, deviceId);
return redisCluster.withCluster(connection -> {
final CompletableFuture<PushRecord> getFuture = connection.async().get(key).toCompletableFuture()
.thenApply(recordJson -> {
if (StringUtils.isNotEmpty(recordJson)) {
try {
return SystemMapper.getMapper().readValue(recordJson, PushRecord.class);
} catch (JsonProcessingException e) {
return null;
}
} else {
return null;
public void recordQueueRead(final UUID accountUuid, final long deviceId, final String userAgent) {
getLatencyAndClearTimestamp(accountUuid, deviceId, System.currentTimeMillis()).thenAccept(latency -> {
if (latency != null) {
Metrics.timer(TIMER_NAME, Tags.of(UserAgentTagUtil.getPlatformTag(userAgent))).record(latency, TimeUnit.MILLISECONDS);
}
});
});
}
getFuture.whenComplete((record, cause) -> {
if (cause == null) {
connection.async().del(key);
}
});
@VisibleForTesting
CompletableFuture<Long> getLatencyAndClearTimestamp(final UUID accountUuid, final long deviceId, final long currentTimeMillis) {
final String key = getFirstUnacknowledgedPushKey(accountUuid, deviceId);
return getFuture;
});
}
return redisCluster.withCluster(connection -> {
final RedisAdvancedClusterAsyncCommands<String, String> commands = connection.async();
private static String getFirstUnacknowledgedPushKey(final UUID accountUuid, final long deviceId) {
return "push_latency::v2::" + accountUuid.toString() + "::" + deviceId;
}
final CompletableFuture<String> getFuture = commands.get(key).toCompletableFuture();
commands.del(key);
return getFuture.thenApply(timestampString -> timestampString != null ? currentTimeMillis - Long.parseLong(timestampString, 10) : null);
});
}
private static String getFirstUnacknowledgedPushKey(final UUID accountUuid, final long deviceId) {
return "push_latency::" + accountUuid.toString() + "::" + deviceId;
}
}
@@ -125,7 +125,7 @@ public class APNSender implements Managed {
private void handleUnregisteredUser(String registrationId, UUID uuid, long deviceId) {
// logger.info("Got APN Unregistered: " + number + "," + deviceId);
Optional<Account> account = accountsManager.getByAccountIdentifier(uuid);
Optional<Account> account = accountsManager.get(uuid);
if (account.isEmpty()) {
logger.info("No account found: {}", uuid);
@@ -99,7 +99,7 @@ public class ApnFallbackManager implements Managed {
final Optional<Account> maybeAccount = separated.map(Pair::first)
.map(UUID::fromString)
.flatMap(accountsManager::getByAccountIdentifier);
.flatMap(accountsManager::get);
final Optional<Device> maybeDevice = separated.map(Pair::second)
.flatMap(deviceId -> maybeAccount.flatMap(account -> account.getDevice(deviceId)));
@@ -171,16 +171,8 @@ public class ClientPresenceManager extends RedisClusterPubSubAdapter<String, Str
}
}
public void disconnectPresence(final UUID accountUuid, final long deviceId) {
final String presenceKey = getPresenceKey(accountUuid, deviceId);
if (isLocallyPresent(accountUuid, deviceId)) {
displacePresence(presenceKey);
}
// If connected locally, we still need to clean up the presence key.
// If connected remotely, the other server will get a keyspace message and handle the disconnect
presenceCluster.useCluster(connection -> connection.sync().del(presenceKey));
public void displacePresence(final UUID accountUuid, final long deviceId) {
displacePresence(getPresenceKey(accountUuid, deviceId));
}
private void displacePresence(final String presenceKey) {
@@ -276,22 +268,18 @@ public class ClientPresenceManager extends RedisClusterPubSubAdapter<String, Str
public void message(final RedisClusterNode node, final String channel, final String message) {
pubSubMessageMeter.mark();
if (channel.startsWith("__keyspace@0__:presence::{")) {
if ("set".equals(message) || "del".equals(message)) {
// for "set", another process has overwritten this presence key, which means the client has connected to another host.
// for "del", another process has indicated the client should be disconnected
// At this point, we're on a Lettuce IO thread and need to dispatch to a separate thread before making
// synchronous Lettuce calls to avoid deadlocking.
keyspaceNotificationExecutorService.execute(() -> {
try {
displacePresence(channel.substring("__keyspace@0__:".length()));
remoteDisplacementMeter.mark();
} catch (final Exception e) {
log.warn("Error displacing presence", e);
}
});
}
if ("set".equals(message) && channel.startsWith("__keyspace@0__:presence::{")) {
// Another process has overwritten this presence key, which means the client has connected to another host.
// At this point, we're on a Lettuce IO thread and need to dispatch to a separate thread before making
// synchronous Lettuce calls to avoid deadlocking.
keyspaceNotificationExecutorService.execute(() -> {
try {
displacePresence(channel.substring("__keyspace@0__:".length()));
remoteDisplacementMeter.mark();
} catch (final Exception e) {
log.warn("Error displacing presence", e);
}
});
}
}
@@ -139,7 +139,7 @@ public class GCMSender {
}
private Optional<Account> getAccountForEvent(GcmMessage message) {
Optional<Account> account = message.getUuid().flatMap(accountsManager::getByAccountIdentifier);
Optional<Account> account = message.getUuid().flatMap(accountsManager::get);
if (account.isPresent()) {
Optional<Device> device = account.get().getDevice(message.getDeviceId());
@@ -124,24 +124,22 @@ public class MessageSender implements Managed {
gcmSender.sendMessage(gcmMessage);
RedisOperation.unchecked(() -> pushLatencyManager.recordPushSent(account.getUuid(), device.getId(), false));
RedisOperation.unchecked(() -> pushLatencyManager.recordPushSent(account.getUuid(), device.getId()));
}
private void sendApnNotification(Account account, Device device) {
ApnMessage apnMessage;
final boolean useVoip = !Util.isEmpty(device.getVoipApnId());
if (useVoip) {
apnMessage = new ApnMessage(device.getVoipApnId(), account.getUuid(), device.getId(), useVoip, Type.NOTIFICATION, Optional.empty());
if (!Util.isEmpty(device.getVoipApnId())) {
apnMessage = new ApnMessage(device.getVoipApnId(), account.getUuid(), device.getId(), true, Type.NOTIFICATION, Optional.empty());
RedisOperation.unchecked(() -> apnFallbackManager.schedule(account, device));
} else {
apnMessage = new ApnMessage(device.getApnId(), account.getUuid(), device.getId(), useVoip, Type.NOTIFICATION, Optional.empty());
apnMessage = new ApnMessage(device.getApnId(), account.getUuid(), device.getId(), false, Type.NOTIFICATION, Optional.empty());
}
apnSender.sendMessage(apnMessage);
RedisOperation.unchecked(() -> pushLatencyManager.recordPushSent(account.getUuid(), device.getId(), useVoip));
RedisOperation.unchecked(() -> pushLatencyManager.recordPushSent(account.getUuid(), device.getId()));
}
@Override
@@ -33,7 +33,7 @@ public class ReceiptSender {
return;
}
final Account destinationAccount = accountManager.getByAccountIdentifier(destinationUuid)
final Account destinationAccount = accountManager.get(destinationUuid)
.orElseThrow(() -> new NoSuchUserException(destinationUuid));
final Envelope.Builder message = Envelope.newBuilder()
@@ -41,7 +41,6 @@ public class ReceiptSender {
.setSource(sourceAccount.getNumber())
.setSourceUuid(sourceAccount.getUuid().toString())
.setSourceDevice((int) source.getAuthenticatedDevice().getId())
.setDestinationUuid(destinationUuid.toString())
.setTimestamp(messageId)
.setType(Envelope.Type.SERVER_DELIVERY_RECEIPT);
@@ -41,7 +41,7 @@ public class LegacyRecaptchaClient implements RecaptchaClient {
.post(Entity.form(formData), VerifyResponse.class);
if (response.success) {
logger.debug("Got successful captcha time: " + response.challenge_ts + ", current time: " + System.currentTimeMillis());
logger.info("Got successful captcha time: " + response.challenge_ts + ", current time: " + System.currentTimeMillis());
}
return response.success;
@@ -23,7 +23,6 @@ import org.whispersystems.textsecuregcm.auth.AuthenticationCredentials;
import org.whispersystems.textsecuregcm.auth.StoredRegistrationLock;
import org.whispersystems.textsecuregcm.entities.AccountAttributes;
import org.whispersystems.textsecuregcm.util.Util;
import javax.annotation.Nullable;
public class Account {
@@ -33,25 +32,15 @@ public class Account {
@JsonIgnore
private UUID uuid;
@JsonProperty("pni")
private UUID phoneNumberIdentifier;
@JsonProperty
private String number;
@JsonProperty
@Nullable
private String username;
@JsonProperty
private Set<Device> devices = new HashSet<>();
@JsonProperty
private String identityKey;
@JsonProperty("pniIdentityKey")
private String phoneNumberIdentityKey;
@JsonProperty("cpv")
private String currentProfileVersion;
@@ -91,10 +80,9 @@ public class Account {
public Account() {}
@VisibleForTesting
public Account(String number, UUID uuid, final UUID phoneNumberIdentifier, Set<Device> devices, byte[] unidentifiedAccessKey) {
public Account(String number, UUID uuid, Set<Device> devices, byte[] unidentifiedAccessKey) {
this.number = number;
this.uuid = uuid;
this.phoneNumberIdentifier = phoneNumberIdentifier;
this.devices = devices;
this.unidentifiedAccessKey = unidentifiedAccessKey;
}
@@ -110,20 +98,16 @@ public class Account {
this.uuid = uuid;
}
public UUID getPhoneNumberIdentifier() {
public void setNumber(String number) {
requireNotStale();
return phoneNumberIdentifier;
this.number = number;
}
/**
* Tests whether this account's account identifier or phone number identifier matches the given UUID.
*
* @param identifier the identifier to test
* @return {@code true} if this account's identifier or phone number identifier matches
*/
public boolean isIdentifiedBy(final UUID identifier) {
return uuid.equals(identifier) || (phoneNumberIdentifier != null && phoneNumberIdentifier.equals(identifier));
public void setCanonicallyDiscoverable(boolean canonicallyDiscoverable) {
requireNotStale();
this.canonicallyDiscoverable = canonicallyDiscoverable;
}
public String getNumber() {
@@ -132,25 +116,6 @@ public class Account {
return number;
}
public void setNumber(String number, UUID phoneNumberIdentifier) {
requireNotStale();
this.number = number;
this.phoneNumberIdentifier = phoneNumberIdentifier;
}
public Optional<String> getUsername() {
requireNotStale();
return Optional.ofNullable(username);
}
public void setUsername(final String username) {
requireNotStale();
this.username = username;
}
public void addDevice(Device device) {
requireNotStale();
@@ -282,12 +247,6 @@ public class Account {
return canonicallyDiscoverable;
}
public void setCanonicallyDiscoverable(boolean canonicallyDiscoverable) {
requireNotStale();
this.canonicallyDiscoverable = canonicallyDiscoverable;
}
public Optional<String> getRelay() {
requireNotStale();
@@ -306,14 +265,6 @@ public class Account {
return identityKey;
}
public String getPhoneNumberIdentityKey() {
return phoneNumberIdentityKey;
}
public void setPhoneNumberIdentityKey(final String phoneNumberIdentityKey) {
this.phoneNumberIdentityKey = phoneNumberIdentityKey;
}
public long getLastSeen() {
requireNotStale();
@@ -530,4 +481,5 @@ public class Account {
logger.error("Accessor called on stale account", new RuntimeException());
}
}
}
@@ -34,7 +34,6 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
private static final long WORKER_TTL_MS = 120_000L;
private static final long ACCELERATED_CHUNK_INTERVAL = 10L;
private final String name;
private final AccountsManager accounts;
private final int chunkSize;
private final long chunkIntervalMs;
@@ -45,13 +44,11 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
private AtomicBoolean running = new AtomicBoolean(false);
private boolean finished;
public AccountDatabaseCrawler(final String name,
AccountsManager accounts,
public AccountDatabaseCrawler(AccountsManager accounts,
AccountDatabaseCrawlerCache cache,
List<AccountDatabaseCrawlerListener> listeners,
int chunkSize,
long chunkIntervalMs) {
this.name = name;
this.accounts = accounts;
this.chunkSize = chunkSize;
this.chunkIntervalMs = chunkIntervalMs;
@@ -85,7 +82,7 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
accelerated = doPeriodicWork();
sleepWhileRunning(accelerated ? ACCELERATED_CHUNK_INTERVAL : chunkIntervalMs);
} catch (Throwable t) {
logger.warn("{}: error in database crawl: {}: {}", name, t.getClass().getSimpleName(), t.getMessage(), t);
logger.warn("error in database crawl: {}: {}", t.getClass().getSimpleName(), t.getMessage(), t);
Util.sleep(10000);
}
}
@@ -109,7 +106,7 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
final long endTimeMs = System.currentTimeMillis();
final long sleepIntervalMs = chunkIntervalMs - (endTimeMs - startTimeMs);
if (sleepIntervalMs > 0) {
logger.debug("{}: Sleeping {}ms", name, sleepIntervalMs);
logger.debug("Sleeping {}ms", sleepIntervalMs);
sleepWhileRunning(sleepIntervalMs);
}
} finally {
@@ -126,19 +123,19 @@ public class AccountDatabaseCrawler implements Managed, Runnable {
final Optional<UUID> fromUuid = getLastUuid();
if (fromUuid.isEmpty()) {
logger.info("{}: Started crawl", name);
logger.info("Started crawl");
listeners.forEach(AccountDatabaseCrawlerListener::onCrawlStart);
}
final AccountCrawlChunk chunkAccounts = readChunk(fromUuid, chunkSize);
if (chunkAccounts.getAccounts().isEmpty()) {
logger.info("{}: Finished crawl", name);
logger.info("Finished crawl");
listeners.forEach(listener -> listener.onCrawlEnd(fromUuid));
cacheLastUuid(Optional.empty());
cache.setAccelerated(false);
} else {
logger.debug("{}: Processing chunk", name);
logger.debug("Processing chunk");
try {
for (AccountDatabaseCrawlerListener listener : listeners) {
listener.timeAndProcessCrawlChunk(fromUuid, chunkAccounts.getAccounts());
@@ -1,5 +1,5 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
@@ -16,9 +16,6 @@ import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
public class AccountDatabaseCrawlerCache {
public static final String GENERAL_PURPOSE_PREFIX = "";
public static final String DIRECTORY_RECONCILER_PREFIX = "directory-reconciler";
private static final String ACTIVE_WORKER_KEY = "account_database_crawler_cache_active_worker";
private static final String LAST_UUID_KEY = "account_database_crawler_cache_last_uuid";
private static final String ACCELERATE_KEY = "account_database_crawler_cache_accelerate";
@@ -30,21 +27,17 @@ public class AccountDatabaseCrawlerCache {
private final FaultTolerantRedisCluster cacheCluster;
private final ClusterLuaScript unlockClusterScript;
private final String prefix;
public AccountDatabaseCrawlerCache(FaultTolerantRedisCluster cacheCluster, String prefix) throws IOException {
public AccountDatabaseCrawlerCache(FaultTolerantRedisCluster cacheCluster) throws IOException {
this.cacheCluster = cacheCluster;
this.unlockClusterScript = ClusterLuaScript.fromResource(cacheCluster, "lua/account_database_crawler/unlock.lua",
ScriptOutputType.INTEGER);
this.prefix = prefix + "::";
}
public void setAccelerated(final boolean accelerated) {
if (accelerated) {
cacheCluster.useCluster(connection -> connection.sync().set(getPrefixedKey(ACCELERATE_KEY), "1"));
cacheCluster.useCluster(connection -> connection.sync().set(ACCELERATE_KEY, "1"));
} else {
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(ACCELERATE_KEY)));
cacheCluster.useCluster(connection -> connection.sync().del(ACCELERATE_KEY));
}
}
@@ -54,16 +47,16 @@ public class AccountDatabaseCrawlerCache {
public boolean claimActiveWork(String workerId, long ttlMs) {
return "OK".equals(cacheCluster.withCluster(connection -> connection.sync()
.set(getPrefixedKey(ACTIVE_WORKER_KEY), workerId, SetArgs.Builder.nx().px(ttlMs))));
.set(ACTIVE_WORKER_KEY, workerId, SetArgs.Builder.nx().px(ttlMs))));
}
public void releaseActiveWork(String workerId) {
unlockClusterScript.execute(List.of(getPrefixedKey(ACTIVE_WORKER_KEY)), List.of(workerId));
unlockClusterScript.execute(List.of(ACTIVE_WORKER_KEY), List.of(workerId));
}
public Optional<UUID> getLastUuid() {
final String lastUuidString = cacheCluster.withCluster(
connection -> connection.sync().get(getPrefixedKey(LAST_UUID_KEY)));
connection -> connection.sync().get(LAST_UUID_KEY));
if (lastUuidString == null) {
return Optional.empty();
@@ -75,15 +68,15 @@ public class AccountDatabaseCrawlerCache {
public void setLastUuid(Optional<UUID> lastUuid) {
if (lastUuid.isPresent()) {
cacheCluster.useCluster(connection -> connection.sync()
.psetex(getPrefixedKey(LAST_UUID_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
.psetex(LAST_UUID_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
} else {
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_KEY)));
cacheCluster.useCluster(connection -> connection.sync().del(LAST_UUID_KEY));
}
}
public Optional<UUID> getLastUuidDynamo() {
final String lastUuidString = cacheCluster.withCluster(
connection -> connection.sync().get(getPrefixedKey(LAST_UUID_DYNAMO_KEY)));
connection -> connection.sync().get(LAST_UUID_DYNAMO_KEY));
if (lastUuidString == null) {
return Optional.empty();
@@ -96,14 +89,9 @@ public class AccountDatabaseCrawlerCache {
if (lastUuid.isPresent()) {
cacheCluster.useCluster(
connection -> connection.sync()
.psetex(getPrefixedKey(LAST_UUID_DYNAMO_KEY), LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
.psetex(LAST_UUID_DYNAMO_KEY, LAST_NUMBER_TTL_MS, lastUuid.get().toString()));
} else {
cacheCluster.useCluster(connection -> connection.sync().del(getPrefixedKey(LAST_UUID_DYNAMO_KEY)));
cacheCluster.useCluster(connection -> connection.sync().del(LAST_UUID_DYNAMO_KEY));
}
}
private String getPrefixedKey(final String key) {
return prefix + key;
}
}
@@ -13,16 +13,11 @@ import io.micrometer.core.instrument.Timer;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
@@ -34,6 +29,7 @@ import software.amazon.awssdk.services.dynamodb.model.Delete;
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
import software.amazon.awssdk.services.dynamodb.model.Put;
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
import software.amazon.awssdk.services.dynamodb.model.ReturnValuesOnConditionCheckFailure;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItem;
@@ -48,8 +44,6 @@ public class Accounts extends AbstractDynamoDbStore {
// uuid, primary key
static final String KEY_ACCOUNT_UUID = "U";
// uuid, attribute on account table, primary key for PNI table
static final String ATTR_PNI_UUID = "PNI";
// phone number
static final String ATTR_ACCOUNT_E164 = "P";
// account, serialized to JSON
@@ -58,44 +52,32 @@ public class Accounts extends AbstractDynamoDbStore {
static final String ATTR_VERSION = "V";
// canonically discoverable
static final String ATTR_CANONICALLY_DISCOVERABLE = "C";
// username; string
static final String ATTR_USERNAME = "N";
private final DynamoDbClient client;
private final String phoneNumberConstraintTableName;
private final String phoneNumberIdentifierConstraintTableName;
private final String usernamesConstraintTableName;
private final String phoneNumbersTableName;
private final String accountsTableName;
private final int scanPageSize;
private static final Timer CREATE_TIMER = Metrics.timer(name(Accounts.class, "create"));
private static final Timer CHANGE_NUMBER_TIMER = Metrics.timer(name(Accounts.class, "changeNumber"));
private static final Timer SET_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "setUsername"));
private static final Timer CLEAR_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "clearUsername"));
private static final Timer UPDATE_TIMER = Metrics.timer(name(Accounts.class, "update"));
private static final Timer GET_BY_NUMBER_TIMER = Metrics.timer(name(Accounts.class, "getByNumber"));
private static final Timer GET_BY_USERNAME_TIMER = Metrics.timer(name(Accounts.class, "getByUsername"));
private static final Timer GET_BY_PNI_TIMER = Metrics.timer(name(Accounts.class, "getByPni"));
private static final Timer GET_BY_UUID_TIMER = Metrics.timer(name(Accounts.class, "getByUuid"));
private static final Timer GET_ALL_FROM_START_TIMER = Metrics.timer(name(Accounts.class, "getAllFrom"));
private static final Timer GET_ALL_FROM_OFFSET_TIMER = Metrics.timer(name(Accounts.class, "getAllFromOffset"));
private static final Timer DELETE_TIMER = Metrics.timer(name(Accounts.class, "delete"));
private static final Logger log = LoggerFactory.getLogger(Accounts.class);
public Accounts(DynamoDbClient client, String accountsTableName, String phoneNumberConstraintTableName,
String phoneNumberIdentifierConstraintTableName, final String usernamesConstraintTableName,
public Accounts(DynamoDbClient client, String accountsTableName, String phoneNumbersTableName,
final int scanPageSize) {
super(client);
this.client = client;
this.phoneNumberConstraintTableName = phoneNumberConstraintTableName;
this.phoneNumberIdentifierConstraintTableName = phoneNumberIdentifierConstraintTableName;
this.phoneNumbersTableName = phoneNumbersTableName;
this.accountsTableName = accountsTableName;
this.usernamesConstraintTableName = usernamesConstraintTableName;
this.scanPageSize = scanPageSize;
}
@@ -103,91 +85,35 @@ public class Accounts extends AbstractDynamoDbStore {
return CREATE_TIMER.record(() -> {
try {
TransactWriteItem phoneNumberConstraintPut = TransactWriteItem.builder()
.put(
Put.builder()
.tableName(phoneNumberConstraintTableName)
.item(Map.of(
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.conditionExpression(
"attribute_not_exists(#number) OR (attribute_exists(#number) AND #uuid = :uuid)")
.expressionAttributeNames(
Map.of("#uuid", KEY_ACCOUNT_UUID,
"#number", ATTR_ACCOUNT_E164))
.expressionAttributeValues(
Map.of(":uuid", AttributeValues.fromUUID(account.getUuid())))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build();
TransactWriteItem phoneNumberIdentifierConstraintPut = TransactWriteItem.builder()
.put(
Put.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.item(Map.of(
ATTR_PNI_UUID, AttributeValues.fromUUID(account.getPhoneNumberIdentifier()),
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.conditionExpression(
"attribute_not_exists(#pni) OR (attribute_exists(#pni) AND #uuid = :uuid)")
.expressionAttributeNames(
Map.of("#uuid", KEY_ACCOUNT_UUID,
"#pni", ATTR_PNI_UUID))
.expressionAttributeValues(
Map.of(":uuid", AttributeValues.fromUUID(account.getUuid())))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build();
final Map<String, AttributeValue> item = new HashMap<>(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid()),
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
ATTR_PNI_UUID, AttributeValues.fromUUID(account.getPhoneNumberIdentifier()),
ATTR_ACCOUNT_DATA, AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
ATTR_VERSION, AttributeValues.fromInt(account.getVersion()),
ATTR_CANONICALLY_DISCOVERABLE, AttributeValues.fromBool(account.shouldBeVisibleInDirectory())));
TransactWriteItem accountPut = TransactWriteItem.builder()
.put(Put.builder()
.conditionExpression("attribute_not_exists(#number) OR #number = :number")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164))
.expressionAttributeValues(Map.of(":number", AttributeValues.fromString(account.getNumber())))
.tableName(accountsTableName)
.item(item)
.build())
.build();
TransactWriteItem phoneNumberConstraintPut = buildPutWriteItemForPhoneNumberConstraint(account, account.getUuid());
TransactWriteItem accountPut = buildPutWriteItemForAccount(account, account.getUuid(), Put.builder()
.conditionExpression("attribute_not_exists(#number) OR #number = :number")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164))
.expressionAttributeValues(Map.of(":number", AttributeValues.fromString(account.getNumber()))));
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(phoneNumberConstraintPut, phoneNumberIdentifierConstraintPut, accountPut)
.transactItems(phoneNumberConstraintPut, accountPut)
.build();
try {
client.transactWriteItems(request);
} catch (TransactionCanceledException e) {
final CancellationReason accountCancellationReason = e.cancellationReasons().get(2);
final CancellationReason accountCancellationReason = e.cancellationReasons().get(1);
if ("ConditionalCheckFailed".equals(accountCancellationReason.code())) {
throw new IllegalArgumentException("account identifier present with different phone number");
throw new IllegalArgumentException("uuid present with different phone number");
}
final CancellationReason phoneNumberConstraintCancellationReason = e.cancellationReasons().get(0);
final CancellationReason phoneNumberIdentifierConstraintCancellationReason = e.cancellationReasons().get(1);
if ("ConditionalCheckFailed".equals(phoneNumberConstraintCancellationReason.code()) ||
"ConditionalCheckFailed".equals(phoneNumberIdentifierConstraintCancellationReason.code())) {
if ("ConditionalCheckFailed".equals(phoneNumberConstraintCancellationReason.code())) {
// In theory, both reasons should trip in tandem and either should give us the information we need. Even so,
// we'll be cautious here and make sure we're choosing a condition check that really failed.
final CancellationReason reason = "ConditionalCheckFailed".equals(phoneNumberConstraintCancellationReason.code()) ?
phoneNumberConstraintCancellationReason : phoneNumberIdentifierConstraintCancellationReason;
ByteBuffer actualAccountUuid = reason.item().get(KEY_ACCOUNT_UUID).b().asByteBuffer();
ByteBuffer actualAccountUuid = phoneNumberConstraintCancellationReason.item().get(KEY_ACCOUNT_UUID).b().asByteBuffer();
account.setUuid(UUIDUtil.fromByteBuffer(actualAccountUuid));
final Account existingAccount = getByAccountIdentifier(account.getUuid()).orElseThrow();
account.setNumber(existingAccount.getNumber(), existingAccount.getPhoneNumberIdentifier());
account.setVersion(existingAccount.getVersion());
final int version = get(account.getUuid()).get().getVersion();
account.setVersion(version);
update(account);
@@ -199,7 +125,7 @@ public class Accounts extends AbstractDynamoDbStore {
throw new ContestedOptimisticLockException();
}
// this shouldn't happen
// this shouldn’t happen
throw new RuntimeException("could not create account: " + extractCancellationReasonCodes(e));
}
} catch (JsonProcessingException e) {
@@ -210,11 +136,44 @@ public class Accounts extends AbstractDynamoDbStore {
});
}
private TransactWriteItem buildPutWriteItemForAccount(Account account, UUID uuid, Put.Builder putBuilder) throws JsonProcessingException {
return TransactWriteItem.builder()
.put(putBuilder
.tableName(accountsTableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid),
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
ATTR_ACCOUNT_DATA, AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
ATTR_VERSION, AttributeValues.fromInt(account.getVersion()),
ATTR_CANONICALLY_DISCOVERABLE, AttributeValues.fromBool(account.shouldBeVisibleInDirectory())))
.build())
.build();
}
private TransactWriteItem buildPutWriteItemForPhoneNumberConstraint(Account account, UUID uuid) {
return TransactWriteItem.builder()
.put(
Put.builder()
.tableName(phoneNumbersTableName)
.item(Map.of(
ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber()),
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid)))
.conditionExpression(
"attribute_not_exists(#number) OR (attribute_exists(#number) AND #uuid = :uuid)")
.expressionAttributeNames(
Map.of("#uuid", KEY_ACCOUNT_UUID,
"#number", ATTR_ACCOUNT_E164))
.expressionAttributeValues(
Map.of(":uuid", AttributeValues.fromUUID(uuid)))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build();
}
/**
* Changes the phone number for the given account. The given account's number should be its current, pre-change
* number. If this method succeeds, the account's number will be changed to the new number and its phone number
* identifier will be changed to the given phone number identifier. If the update fails for any reason, the account's
* number and PNI will be unchanged.
* number. If this method succeeds, the account's number will be changed to the new number. If the update fails for
* any reason, the account's number will be unchanged.
* <p/>
* This method expects that any accounts with conflicting numbers will have been removed by the time this method is
* called. This method may fail with an unspecified {@link RuntimeException} if another account with the same number
@@ -223,28 +182,26 @@ public class Accounts extends AbstractDynamoDbStore {
* @param account the account for which to change the phone number
* @param number the new phone number
*/
public void changeNumber(final Account account, final String number, final UUID phoneNumberIdentifier) {
public void changeNumber(final Account account, final String number) {
CHANGE_NUMBER_TIMER.record(() -> {
final String originalNumber = account.getNumber();
final UUID originalPni = account.getPhoneNumberIdentifier();
boolean succeeded = false;
account.setNumber(number, phoneNumberIdentifier);
account.setNumber(number);
try {
final List<TransactWriteItem> writeItems = new ArrayList<>();
writeItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(phoneNumberConstraintTableName)
.tableName(phoneNumbersTableName)
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(originalNumber)))
.build())
.build());
writeItems.add(TransactWriteItem.builder()
.put(Put.builder()
.tableName(phoneNumberConstraintTableName)
.tableName(phoneNumbersTableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid()),
ATTR_ACCOUNT_E164, AttributeValues.fromString(number)))
@@ -254,41 +211,20 @@ public class Accounts extends AbstractDynamoDbStore {
.build())
.build());
writeItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.key(Map.of(ATTR_PNI_UUID, AttributeValues.fromUUID(originalPni)))
.build())
.build());
writeItems.add(TransactWriteItem.builder()
.put(Put.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid()),
ATTR_PNI_UUID, AttributeValues.fromUUID(phoneNumberIdentifier)))
.conditionExpression("attribute_not_exists(#pni)")
.expressionAttributeNames(Map.of("#pni", ATTR_PNI_UUID))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build());
writeItems.add(
TransactWriteItem.builder()
.update(Update.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.updateExpression("SET #data = :data, #number = :number, #pni = :pni, #cds = :cds ADD #version :version_increment")
.updateExpression("SET #data = :data, #number = :number, #cds = :cds ADD #version :version_increment")
.conditionExpression("attribute_exists(#number) AND #version = :version")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164,
"#data", ATTR_ACCOUNT_DATA,
"#cds", ATTR_CANONICALLY_DISCOVERABLE,
"#pni", ATTR_PNI_UUID,
"#version", ATTR_VERSION))
.expressionAttributeValues(Map.of(
":data", AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
":number", AttributeValues.fromString(number),
":pni", AttributeValues.fromUUID(phoneNumberIdentifier),
":cds", AttributeValues.fromBool(account.shouldBeVisibleInDirectory()),
":version", AttributeValues.fromInt(account.getVersion()),
":version_increment", AttributeValues.fromInt(1)))
@@ -307,227 +243,65 @@ public class Accounts extends AbstractDynamoDbStore {
throw new IllegalArgumentException(e);
} finally {
if (!succeeded) {
account.setNumber(originalNumber, originalPni);
account.setNumber(originalNumber);
}
}
});
}
public void setUsername(final Account account, final String username)
throws ContestedOptimisticLockException, UsernameNotAvailableException {
final long startNanos = System.nanoTime();
final Optional<String> maybeOriginalUsername = account.getUsername();
account.setUsername(username);
boolean succeeded = false;
try {
final List<TransactWriteItem> writeItems = new ArrayList<>();
writeItems.add(TransactWriteItem.builder()
.put(Put.builder()
.tableName(usernamesConstraintTableName)
.item(Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid()),
ATTR_USERNAME, AttributeValues.fromString(username)))
.conditionExpression("attribute_not_exists(#username)")
.expressionAttributeNames(Map.of("#username", ATTR_USERNAME))
.returnValuesOnConditionCheckFailure(ReturnValuesOnConditionCheckFailure.ALL_OLD)
.build())
.build());
writeItems.add(
TransactWriteItem.builder()
.update(Update.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.updateExpression("SET #data = :data, #username = :username ADD #version :version_increment")
.conditionExpression("#version = :version")
.expressionAttributeNames(Map.of("#data", ATTR_ACCOUNT_DATA,
"#username", ATTR_USERNAME,
"#version", ATTR_VERSION))
.expressionAttributeValues(Map.of(
":data", AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
":username", AttributeValues.fromString(username),
":version", AttributeValues.fromInt(account.getVersion()),
":version_increment", AttributeValues.fromInt(1)))
.build())
.build());
maybeOriginalUsername.ifPresent(originalUsername -> writeItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(usernamesConstraintTableName)
.key(Map.of(ATTR_USERNAME, AttributeValues.fromString(originalUsername)))
.build())
.build()));
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(writeItems)
.build();
client.transactWriteItems(request);
account.setVersion(account.getVersion() + 1);
succeeded = true;
} catch (final JsonProcessingException e) {
throw new IllegalArgumentException(e);
} catch (final TransactionCanceledException e) {
if ("ConditionalCheckFailed".equals(e.cancellationReasons().get(0).code())) {
throw new UsernameNotAvailableException();
} else if ("ConditionalCheckFailed".equals(e.cancellationReasons().get(1).code())) {
throw new ContestedOptimisticLockException();
}
throw e;
} finally {
if (!succeeded) {
account.setUsername(maybeOriginalUsername.orElse(null));
}
SET_USERNAME_TIMER.record(System.nanoTime() - startNanos, TimeUnit.NANOSECONDS);
}
}
public void clearUsername(Account account) {
account.getUsername().ifPresent(username -> {
CLEAR_USERNAME_TIMER.record(() -> {
account.setUsername(null);
boolean succeeded = false;
try {
final List<TransactWriteItem> writeItems = new ArrayList<>();
writeItems.add(
TransactWriteItem.builder()
.update(Update.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.updateExpression("SET #data = :data REMOVE #username ADD #version :version_increment")
.conditionExpression("#version = :version")
.expressionAttributeNames(Map.of("#data", ATTR_ACCOUNT_DATA,
"#username", ATTR_USERNAME,
"#version", ATTR_VERSION))
.expressionAttributeValues(Map.of(
":data", AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
":version", AttributeValues.fromInt(account.getVersion()),
":version_increment", AttributeValues.fromInt(1)))
.build())
.build());
writeItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(usernamesConstraintTableName)
.key(Map.of(ATTR_USERNAME, AttributeValues.fromString(username)))
.build())
.build());
final TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(writeItems)
.build();
client.transactWriteItems(request);
account.setVersion(account.getVersion() + 1);
succeeded = true;
} catch (final JsonProcessingException e) {
throw new IllegalArgumentException(e);
} catch (final TransactionCanceledException e) {
if ("ConditionalCheckFailed".equals(e.cancellationReasons().get(0).code())) {
throw new ContestedOptimisticLockException();
}
throw e;
} finally {
if (!succeeded) {
account.setUsername(username);
}
}
});
});
}
public void update(Account account) throws ContestedOptimisticLockException {
UPDATE_TIMER.record(() -> {
final UpdateItemRequest updateItemRequest;
UpdateItemRequest updateItemRequest;
try {
updateItemRequest = UpdateItemRequest.builder()
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.updateExpression("SET #data = :data, #cds = :cds ADD #version :version_increment")
.conditionExpression("attribute_exists(#number) AND #version = :version")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164,
"#data", ATTR_ACCOUNT_DATA,
"#cds", ATTR_CANONICALLY_DISCOVERABLE,
"#version", ATTR_VERSION))
.expressionAttributeValues(Map.of(
":data", AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
":cds", AttributeValues.fromBool(account.shouldBeVisibleInDirectory()),
":version", AttributeValues.fromInt(account.getVersion()),
":version_increment", AttributeValues.fromInt(1)))
.build();
.tableName(accountsTableName)
.key(Map.of(KEY_ACCOUNT_UUID, AttributeValues.fromUUID(account.getUuid())))
.updateExpression("SET #data = :data, #cds = :cds ADD #version :version_increment")
.conditionExpression("attribute_exists(#number) AND #version = :version")
.expressionAttributeNames(Map.of("#number", ATTR_ACCOUNT_E164,
"#data", ATTR_ACCOUNT_DATA,
"#cds", ATTR_CANONICALLY_DISCOVERABLE,
"#version", ATTR_VERSION))
.expressionAttributeValues(Map.of(
":data", AttributeValues.fromByteArray(SystemMapper.getMapper().writeValueAsBytes(account)),
":cds", AttributeValues.fromBool(account.shouldBeVisibleInDirectory()),
":version", AttributeValues.fromInt(account.getVersion()),
":version_increment", AttributeValues.fromInt(1)))
.returnValues(ReturnValue.UPDATED_NEW)
.build();
} catch (JsonProcessingException e) {
throw new IllegalArgumentException(e);
}
try {
final UpdateItemResponse response = client.updateItem(updateItemRequest);
UpdateItemResponse response = client.updateItem(updateItemRequest);
account.setVersion(AttributeValues.getInt(response.attributes(), "V", account.getVersion() + 1));
} catch (final TransactionConflictException e) {
throw new ContestedOptimisticLockException();
} catch (final ConditionalCheckFailedException e) {
// the exception doesn't give details about which condition failed,
// the exception doesn’t give details about which condition failed,
// but we can infer it was an optimistic locking failure if the UUID is known
throw getByAccountIdentifier(account.getUuid()).isPresent() ? new ContestedOptimisticLockException() : e;
throw get(account.getUuid()).isPresent() ? new ContestedOptimisticLockException() : e;
}
});
}
public Optional<Account> getByE164(String number) {
public Optional<Account> get(String number) {
return GET_BY_NUMBER_TIMER.record(() -> {
final GetItemResponse response = client.getItem(GetItemRequest.builder()
.tableName(phoneNumberConstraintTableName)
.tableName(phoneNumbersTableName)
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(number)))
.build());
return Optional.ofNullable(response.item())
.map(item -> item.get(KEY_ACCOUNT_UUID))
.map(this::accountByUuid)
.map(Accounts::fromItem);
});
}
public Optional<Account> getByUsername(final String username) {
return GET_BY_USERNAME_TIMER.record(() -> {
final GetItemResponse response = client.getItem(GetItemRequest.builder()
.tableName(usernamesConstraintTableName)
.key(Map.of(ATTR_USERNAME, AttributeValues.fromString(username)))
.build());
return Optional.ofNullable(response.item())
.map(item -> item.get(KEY_ACCOUNT_UUID))
.map(this::accountByUuid)
.map(Accounts::fromItem);
});
}
public Optional<Account> getByPhoneNumberIdentifier(final UUID phoneNumberIdentifier) {
return GET_BY_PNI_TIMER.record(() -> {
final GetItemResponse response = client.getItem(GetItemRequest.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.key(Map.of(ATTR_PNI_UUID, AttributeValues.fromUUID(phoneNumberIdentifier)))
.build());
return Optional.ofNullable(response.item())
.map(item -> item.get(KEY_ACCOUNT_UUID))
.map(this::accountByUuid)
.map(uuid -> accountByUuid(uuid))
.map(Accounts::fromItem);
});
}
@@ -541,7 +315,7 @@ public class Accounts extends AbstractDynamoDbStore {
return r.item().isEmpty() ? null : r.item();
}
public Optional<Account> getByAccountIdentifier(UUID uuid) {
public Optional<Account> get(UUID uuid) {
return GET_BY_UUID_TIMER.record(() ->
Optional.ofNullable(accountByUuid(AttributeValues.fromUUID(uuid)))
.map(Accounts::fromItem));
@@ -550,11 +324,13 @@ public class Accounts extends AbstractDynamoDbStore {
public void delete(UUID uuid) {
DELETE_TIMER.record(() -> {
getByAccountIdentifier(uuid).ifPresent(account -> {
Optional<Account> maybeAccount = get(uuid);
maybeAccount.ifPresent(account -> {
TransactWriteItem phoneNumberDelete = TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(phoneNumberConstraintTableName)
.tableName(phoneNumbersTableName)
.key(Map.of(ATTR_ACCOUNT_E164, AttributeValues.fromString(account.getNumber())))
.build())
.build();
@@ -566,24 +342,8 @@ public class Accounts extends AbstractDynamoDbStore {
.build())
.build();
final List<TransactWriteItem> transactWriteItems = new ArrayList<>(List.of(phoneNumberDelete, accountDelete));
transactWriteItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(phoneNumberIdentifierConstraintTableName)
.key(Map.of(ATTR_PNI_UUID, AttributeValues.fromUUID(account.getPhoneNumberIdentifier())))
.build())
.build());
account.getUsername().ifPresent(username -> transactWriteItems.add(TransactWriteItem.builder()
.delete(Delete.builder()
.tableName(usernamesConstraintTableName)
.key(Map.of(ATTR_USERNAME, AttributeValues.fromString(username)))
.build())
.build()));
TransactWriteItemsRequest request = TransactWriteItemsRequest.builder()
.transactItems(transactWriteItems).build();
.transactItems(phoneNumberDelete, accountDelete).build();
client.transactWriteItems(request);
});
@@ -633,20 +393,8 @@ public class Accounts extends AbstractDynamoDbStore {
}
try {
Account account = SystemMapper.getMapper().readValue(item.get(ATTR_ACCOUNT_DATA).b().asByteArray(), Account.class);
final UUID accountIdentifier = UUIDUtil.fromByteBuffer(item.get(KEY_ACCOUNT_UUID).b().asByteBuffer());
final UUID phoneNumberIdentifierFromAttribute = AttributeValues.getUUID(item, ATTR_PNI_UUID, null);
if (account.getPhoneNumberIdentifier() == null || phoneNumberIdentifierFromAttribute == null ||
!Objects.equals(account.getPhoneNumberIdentifier(), phoneNumberIdentifierFromAttribute)) {
log.warn("Missing or mismatched PNIs for account {}. From JSON: {}; from attribute: {}",
accountIdentifier, account.getPhoneNumberIdentifier(), phoneNumberIdentifierFromAttribute);
}
account.setNumber(item.get(ATTR_ACCOUNT_E164).s(), phoneNumberIdentifierFromAttribute);
account.setUuid(accountIdentifier);
account.setUsername(AttributeValues.getString(item, ATTR_USERNAME, null));
account.setNumber(item.get(ATTR_ACCOUNT_E164).s());
account.setUuid(UUIDUtil.fromByteBuffer(item.get(KEY_ACCOUNT_UUID).b().asByteBuffer()));
account.setVersion(Integer.parseInt(item.get(ATTR_VERSION).n()));
account.setCanonicallyDiscoverable(Optional.ofNullable(item.get(ATTR_CANONICALLY_DISCOVERABLE)).map(av -> av.bool()).orElse(false));
@@ -17,10 +17,6 @@ import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tags;
import java.io.IOException;
import java.time.Clock;
import java.time.Duration;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@@ -40,25 +36,21 @@ import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient;
import org.whispersystems.textsecuregcm.sqs.DirectoryQueue;
import org.whispersystems.textsecuregcm.util.Constants;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import org.whispersystems.textsecuregcm.util.UsernameValidator;
import org.whispersystems.textsecuregcm.util.Util;
public class AccountsManager {
private static final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private static final Timer createTimer = metricRegistry.timer(name(AccountsManager.class, "create"));
private static final Timer updateTimer = metricRegistry.timer(name(AccountsManager.class, "update"));
private static final Timer getByNumberTimer = metricRegistry.timer(name(AccountsManager.class, "getByNumber"));
private static final Timer getByUsernameTimer = metricRegistry.timer(name(AccountsManager.class, "getByUsername"));
private static final Timer getByUuidTimer = metricRegistry.timer(name(AccountsManager.class, "getByUuid"));
private static final Timer deleteTimer = metricRegistry.timer(name(AccountsManager.class, "delete"));
private static final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private static final Timer createTimer = metricRegistry.timer(name(AccountsManager.class, "create" ));
private static final Timer updateTimer = metricRegistry.timer(name(AccountsManager.class, "update" ));
private static final Timer getByNumberTimer = metricRegistry.timer(name(AccountsManager.class, "getByNumber"));
private static final Timer getByUuidTimer = metricRegistry.timer(name(AccountsManager.class, "getByUuid" ));
private static final Timer deleteTimer = metricRegistry.timer(name(AccountsManager.class, "delete"));
private static final Timer redisSetTimer = metricRegistry.timer(name(AccountsManager.class, "redisSet"));
private static final Timer redisSetTimer = metricRegistry.timer(name(AccountsManager.class, "redisSet" ));
private static final Timer redisNumberGetTimer = metricRegistry.timer(name(AccountsManager.class, "redisNumberGet"));
private static final Timer redisUsernameGetTimer = metricRegistry.timer(name(AccountsManager.class, "redisUsernameGet"));
private static final Timer redisPniGetTimer = metricRegistry.timer(name(AccountsManager.class, "redisPniGet"));
private static final Timer redisUuidGetTimer = metricRegistry.timer(name(AccountsManager.class, "redisUuidGet"));
private static final Timer redisDeleteTimer = metricRegistry.timer(name(AccountsManager.class, "redisDelete"));
private static final Timer redisUuidGetTimer = metricRegistry.timer(name(AccountsManager.class, "redisUuidGet" ));
private static final Timer redisDeleteTimer = metricRegistry.timer(name(AccountsManager.class, "redisDelete" ));
private static final String CREATE_COUNTER_NAME = name(AccountsManager.class, "createCounter");
private static final String DELETE_COUNTER_NAME = name(AccountsManager.class, "deleteCounter");
@@ -68,31 +60,18 @@ public class AccountsManager {
private final Logger logger = LoggerFactory.getLogger(AccountsManager.class);
private final Accounts accounts;
private final PhoneNumberIdentifiers phoneNumberIdentifiers;
private final FaultTolerantRedisCluster cacheCluster;
private final DeletedAccountsManager deletedAccountsManager;
private final DirectoryQueue directoryQueue;
private final Keys keys;
private final DirectoryQueue directoryQueue;
private final KeysDynamoDb keysDynamoDb;
private final MessagesManager messagesManager;
private final ReservedUsernames reservedUsernames;
private final ProfilesManager profilesManager;
private final UsernamesManager usernamesManager;
private final ProfilesManager profilesManager;
private final StoredVerificationCodeManager pendingAccounts;
private final SecureStorageClient secureStorageClient;
private final SecureBackupClient secureBackupClient;
private final SecureStorageClient secureStorageClient;
private final SecureBackupClient secureBackupClient;
private final ClientPresenceManager clientPresenceManager;
private final ObjectMapper mapper;
private final Clock clock;
// An account that's used at least daily will get reset in the cache at least once per day when its "last seen"
// timestamp updates; expiring entries after two days will help clear out "zombie" cache entries that are read
// frequently (e.g. the account is in an active group and receives messages frequently), but aren't actively used by
// the owner.
private static final long CACHE_TTL_SECONDS = Duration.ofDays(2).toSeconds();
@FunctionalInterface
private interface AccountPersister {
void persistAccount(Account account) throws UsernameNotAvailableException;
}
private final ObjectMapper mapper;
public enum DeletionReason {
ADMIN_DELETED("admin"),
@@ -106,42 +85,35 @@ public class AccountsManager {
}
}
public AccountsManager(final Accounts accounts,
final PhoneNumberIdentifiers phoneNumberIdentifiers,
final FaultTolerantRedisCluster cacheCluster,
public AccountsManager(Accounts accounts, FaultTolerantRedisCluster cacheCluster,
final DeletedAccountsManager deletedAccountsManager,
final DirectoryQueue directoryQueue,
final Keys keys,
final MessagesManager messagesManager,
final ReservedUsernames reservedUsernames,
final KeysDynamoDb keysDynamoDb, final MessagesManager messagesManager,
final UsernamesManager usernamesManager,
final ProfilesManager profilesManager,
final StoredVerificationCodeManager pendingAccounts,
final SecureStorageClient secureStorageClient,
final SecureBackupClient secureBackupClient,
final ClientPresenceManager clientPresenceManager,
final Clock clock) {
final ClientPresenceManager clientPresenceManager) {
this.accounts = accounts;
this.phoneNumberIdentifiers = phoneNumberIdentifiers;
this.cacheCluster = cacheCluster;
this.deletedAccountsManager = deletedAccountsManager;
this.directoryQueue = directoryQueue;
this.keys = keys;
this.keysDynamoDb = keysDynamoDb;
this.messagesManager = messagesManager;
this.usernamesManager = usernamesManager;
this.profilesManager = profilesManager;
this.pendingAccounts = pendingAccounts;
this.secureStorageClient = secureStorageClient;
this.secureBackupClient = secureBackupClient;
this.clientPresenceManager = clientPresenceManager;
this.reservedUsernames = reservedUsernames;
this.mapper = SystemMapper.getMapper();
this.clock = Objects.requireNonNull(clock);
}
public Account create(final String number,
final String password,
final String signalAgent,
final AccountAttributes accountAttributes,
final List<AccountBadge> accountBadges) throws InterruptedException {
final AccountAttributes accountAttributes) throws InterruptedException {
try (Timer.Context ignored = createTimer.time()) {
final Account account = new Account();
@@ -158,18 +130,17 @@ public class AccountsManager {
device.setLastSeen(Util.todayInMillis());
device.setUserAgent(signalAgent);
account.setNumber(number, phoneNumberIdentifiers.getPhoneNumberIdentifier(number));
account.setNumber(number);
account.setUuid(maybeRecentlyDeletedUuid.orElseGet(UUID::randomUUID));
account.addDevice(device);
account.setRegistrationLockFromAttributes(accountAttributes);
account.setUnidentifiedAccessKey(accountAttributes.getUnidentifiedAccessKey());
account.setUnrestrictedUnidentifiedAccess(accountAttributes.isUnrestrictedUnidentifiedAccess());
account.setDiscoverableByPhoneNumber(accountAttributes.isDiscoverableByPhoneNumber());
account.setBadges(clock, accountBadges);
final UUID originalUuid = account.getUuid();
boolean freshUser = accounts.create(account);
boolean freshUser = dynamoCreate(account);
// create() sometimes updates the UUID, if there was a number conflict.
// for metrics, we want secondary to run with the same original UUID
@@ -193,7 +164,7 @@ public class AccountsManager {
// account and need to clear out messages and keys that may have been stored for the old account.
if (!originalUuid.equals(actualUuid)) {
messagesManager.clear(actualUuid);
keys.delete(actualUuid);
keysDynamoDb.delete(actualUuid);
profilesManager.deleteAll(actualUuid);
}
@@ -231,7 +202,7 @@ public class AccountsManager {
deletedAccountsManager.lockAndPut(account.getNumber(), number, () -> {
redisDelete(account);
final Optional<Account> maybeExistingAccount = getByE164(number);
final Optional<Account> maybeExistingAccount = get(number);
final Optional<UUID> displacedUuid;
if (maybeExistingAccount.isPresent()) {
@@ -242,20 +213,12 @@ public class AccountsManager {
}
final UUID uuid = account.getUuid();
final UUID phoneNumberIdentifier = phoneNumberIdentifiers.getPhoneNumberIdentifier(number);
final Account numberChangedAccount;
try {
numberChangedAccount = updateWithRetries(
account,
a -> true,
a -> accounts.changeNumber(a, number, phoneNumberIdentifier),
() -> accounts.getByAccountIdentifier(uuid).orElseThrow());
} catch (UsernameNotAvailableException e) {
// This should never happen when changing numbers
throw new RuntimeException(e);
}
final Account numberChangedAccount = updateWithRetries(
account,
a -> true,
a -> dynamoChangeNumber(a, number),
() -> dynamoGet(uuid).orElseThrow());
updatedAccount.set(numberChangedAccount);
directoryQueue.changePhoneNumber(numberChangedAccount, originalNumber, number);
@@ -266,53 +229,13 @@ public class AccountsManager {
return updatedAccount.get();
}
public Account setUsername(final Account account, final String username) throws UsernameNotAvailableException {
final String canonicalUsername = UsernameValidator.getCanonicalUsername(username);
if (account.getUsername().map(canonicalUsername::equals).orElse(false)) {
return account;
}
if (reservedUsernames.isReserved(canonicalUsername, account.getUuid())) {
throw new UsernameNotAvailableException();
}
redisDelete(account);
return updateWithRetries(
account,
a -> true,
a -> accounts.setUsername(a, canonicalUsername),
() -> accounts.getByAccountIdentifier(account.getUuid()).orElseThrow());
}
public Account clearUsername(final Account account) {
redisDelete(account);
try {
return updateWithRetries(
account,
a -> true,
accounts::clearUsername,
() -> accounts.getByAccountIdentifier(account.getUuid()).orElseThrow());
} catch (UsernameNotAvailableException e) {
// This should never happen
throw new RuntimeException(e);
}
}
public Account update(Account account, Consumer<Account> updater) {
try {
return update(account, a -> {
updater.accept(a);
// assume that all updaters passed to the public method actually modify the account
return true;
});
} catch (UsernameNotAvailableException e) {
// This should never happen for general-purpose, public account updates
throw new RuntimeException(e);
}
return update(account, a -> {
updater.accept(a);
// assume that all updaters passed to the public method actually modify the account
return true;
});
}
/**
@@ -321,33 +244,28 @@ public class AccountsManager {
*/
public Account updateDeviceLastSeen(Account account, Device device, final long lastSeen) {
try {
return update(account, a -> {
return update(account, a -> {
final Optional<Device> maybeDevice = a.getDevice(device.getId());
final Optional<Device> maybeDevice = a.getDevice(device.getId());
return maybeDevice.map(d -> {
if (d.getLastSeen() >= lastSeen) {
return false;
}
return maybeDevice.map(d -> {
if (d.getLastSeen() >= lastSeen) {
return false;
}
d.setLastSeen(lastSeen);
d.setLastSeen(lastSeen);
return true;
return true;
}).orElse(false);
});
} catch (UsernameNotAvailableException e) {
// This should never happen when updating last-seen timestamps
throw new RuntimeException(e);
}
}).orElse(false);
});
}
/**
* @param account account to update
* @param updater must return {@code true} if the account was actually updated
*/
private Account update(Account account, Function<Account, Boolean> updater) throws UsernameNotAvailableException {
private Account update(Account account, Function<Account, Boolean> updater) {
final boolean wasVisibleBeforeUpdate = account.shouldBeVisibleInDirectory();
@@ -359,13 +277,8 @@ public class AccountsManager {
final UUID uuid = account.getUuid();
final String originalNumber = account.getNumber();
final UUID originalPhoneNumberIdentifier = account.getPhoneNumberIdentifier();
final Optional<String> originalUsername = account.getUsername();
updatedAccount = updateWithRetries(account,
updater,
accounts::update,
() -> accounts.getByAccountIdentifier(uuid).orElseThrow());
updatedAccount = updateWithRetries(account, updater, this::dynamoUpdate, () -> dynamoGet(uuid).get());
assert updatedAccount.getNumber().equals(originalNumber);
@@ -374,20 +287,6 @@ public class AccountsManager {
new RuntimeException());
}
assert updatedAccount.getPhoneNumberIdentifier().equals(originalPhoneNumberIdentifier);
if (!updatedAccount.getPhoneNumberIdentifier().equals(originalPhoneNumberIdentifier)) {
logger.error("Phone number identifier changed via \"normal\" update; PNIs must be changed via changeNumber method",
new RuntimeException());
}
assert updatedAccount.getUsername().equals(originalUsername);
if (!updatedAccount.getUsername().equals(originalUsername)) {
logger.error("Username changed via \"normal\" update; usernames must be changed via setUsername method",
new RuntimeException());
}
redisSet(updatedAccount);
}
@@ -400,8 +299,8 @@ public class AccountsManager {
return updatedAccount;
}
private Account updateWithRetries(Account account, Function<Account, Boolean> updater, AccountPersister persister,
Supplier<Account> retriever) throws UsernameNotAvailableException {
private Account updateWithRetries(Account account, Function<Account, Boolean> updater, Consumer<Account> persister,
Supplier<Account> retriever) {
if (!updater.apply(account)) {
return account;
@@ -413,7 +312,7 @@ public class AccountsManager {
while (tries < maxTries) {
try {
persister.persistAccount(account);
persister.accept(account);
final Account updatedAccount;
try {
@@ -441,24 +340,19 @@ public class AccountsManager {
}
public Account updateDevice(Account account, long deviceId, Consumer<Device> deviceUpdater) {
try {
return update(account, a -> {
a.getDevice(deviceId).ifPresent(deviceUpdater);
// assume that all updaters passed to the public method actually modify the device
return true;
});
} catch (UsernameNotAvailableException e) {
// This should never happen when updating devices
throw new RuntimeException(e);
}
return update(account, a -> {
a.getDevice(deviceId).ifPresent(deviceUpdater);
// assume that all updaters passed to the public method actually modify the device
return true;
});
}
public Optional<Account> getByE164(String number) {
public Optional<Account> get(String number) {
try (Timer.Context ignored = getByNumberTimer.time()) {
Optional<Account> account = redisGetByE164(number);
Optional<Account> account = redisGet(number);
if (account.isEmpty()) {
account = accounts.getByE164(number);
account = dynamoGet(number);
account.ifPresent(this::redisSet);
}
@@ -466,40 +360,12 @@ public class AccountsManager {
}
}
public Optional<Account> getByPhoneNumberIdentifier(UUID pni) {
try (Timer.Context ignored = getByNumberTimer.time()) {
Optional<Account> account = redisGetByPhoneNumberIdentifier(pni);
if (account.isEmpty()) {
account = accounts.getByPhoneNumberIdentifier(pni);
account.ifPresent(this::redisSet);
}
return account;
}
}
public Optional<Account> getByUsername(final String username) {
try (final Timer.Context ignored = getByUsernameTimer.time()) {
final String canonicalUsername = UsernameValidator.getCanonicalUsername(username);
Optional<Account> account = redisGetByUsername(canonicalUsername);
if (account.isEmpty()) {
account = accounts.getByUsername(canonicalUsername);
account.ifPresent(this::redisSet);
}
return account;
}
}
public Optional<Account> getByAccountIdentifier(UUID uuid) {
public Optional<Account> get(UUID uuid) {
try (Timer.Context ignored = getByUuidTimer.time()) {
Optional<Account> account = redisGetByAccountIdentifier(uuid);
Optional<Account> account = redisGet(uuid);
if (account.isEmpty()) {
account = accounts.getByAccountIdentifier(uuid);
account = dynamoGet(uuid);
account.ifPresent(this::redisSet);
}
@@ -538,25 +404,24 @@ public class AccountsManager {
final CompletableFuture<Void> deleteStorageServiceDataFuture = secureStorageClient.deleteStoredData(account.getUuid());
final CompletableFuture<Void> deleteBackupServiceDataFuture = secureBackupClient.deleteBackups(account.getUuid());
usernamesManager.delete(account.getUuid());
profilesManager.deleteAll(account.getUuid());
keys.delete(account.getUuid());
keys.delete(account.getPhoneNumberIdentifier());
keysDynamoDb.delete(account.getUuid());
messagesManager.clear(account.getUuid());
messagesManager.clear(account.getPhoneNumberIdentifier());
deleteStorageServiceDataFuture.join();
deleteBackupServiceDataFuture.join();
accounts.delete(account.getUuid());
redisDelete(account);
dynamoDelete(account);
RedisOperation.unchecked(() ->
account.getDevices().forEach(device ->
clientPresenceManager.disconnectPresence(account.getUuid(), device.getId())));
clientPresenceManager.displacePresence(account.getUuid(), device.getId())));
}
private String getAccountMapKey(String key) {
return "AccountMap::" + key;
private String getAccountMapKey(String number) {
return "AccountMap::" + number;
}
private String getAccountEntityKey(UUID uuid) {
@@ -570,35 +435,19 @@ public class AccountsManager {
cacheCluster.useCluster(connection -> {
final RedisAdvancedClusterCommands<String, String> commands = connection.sync();
commands.setex(getAccountMapKey(account.getPhoneNumberIdentifier().toString()), CACHE_TTL_SECONDS, account.getUuid().toString());
commands.setex(getAccountMapKey(account.getNumber()), CACHE_TTL_SECONDS, account.getUuid().toString());
commands.setex(getAccountEntityKey(account.getUuid()), CACHE_TTL_SECONDS, accountJson);
account.getUsername().ifPresent(username ->
commands.setex(getAccountMapKey(username), CACHE_TTL_SECONDS, account.getUuid().toString()));
commands.set(getAccountMapKey(account.getNumber()), account.getUuid().toString());
commands.set(getAccountEntityKey(account.getUuid()), accountJson);
});
} catch (JsonProcessingException e) {
throw new IllegalStateException(e);
}
}
private Optional<Account> redisGetByPhoneNumberIdentifier(UUID uuid) {
return redisGetBySecondaryKey(uuid.toString(), redisPniGetTimer);
}
private Optional<Account> redisGet(String number) {
try (Timer.Context ignored = redisNumberGetTimer.time()) {
final String uuid = cacheCluster.withCluster(connection -> connection.sync().get(getAccountMapKey(number)));
private Optional<Account> redisGetByE164(String e164) {
return redisGetBySecondaryKey(e164, redisNumberGetTimer);
}
private Optional<Account> redisGetByUsername(String username) {
return redisGetBySecondaryKey(username, redisUsernameGetTimer);
}
private Optional<Account> redisGetBySecondaryKey(String secondaryKey, Timer timer) {
try (Timer.Context ignored = timer.time()) {
final String uuid = cacheCluster.withCluster(connection -> connection.sync().get(getAccountMapKey(secondaryKey)));
if (uuid != null) return redisGetByAccountIdentifier(UUID.fromString(uuid));
if (uuid != null) return redisGet(UUID.fromString(uuid));
else return Optional.empty();
} catch (IllegalArgumentException e) {
logger.warn("Deserialization error", e);
@@ -609,7 +458,7 @@ public class AccountsManager {
}
}
private Optional<Account> redisGetByAccountIdentifier(UUID uuid) {
private Optional<Account> redisGet(UUID uuid) {
try (Timer.Context ignored = redisUuidGetTimer.time()) {
final String json = cacheCluster.withCluster(connection -> connection.sync().get(getAccountEntityKey(uuid)));
@@ -617,10 +466,6 @@ public class AccountsManager {
Account account = mapper.readValue(json, Account.class);
account.setUuid(uuid);
if (account.getPhoneNumberIdentifier() == null) {
logger.warn("Account {} loaded from Redis is missing a PNI", uuid);
}
return Optional.of(account);
}
@@ -636,14 +481,32 @@ public class AccountsManager {
private void redisDelete(final Account account) {
try (final Timer.Context ignored = redisDeleteTimer.time()) {
cacheCluster.useCluster(connection -> {
connection.sync().del(
getAccountMapKey(account.getNumber()),
getAccountMapKey(account.getPhoneNumberIdentifier().toString()),
getAccountEntityKey(account.getUuid()));
account.getUsername().ifPresent(username -> connection.sync().del(getAccountMapKey(username)));
});
cacheCluster.useCluster(connection -> connection.sync()
.del(getAccountMapKey(account.getNumber()), getAccountEntityKey(account.getUuid())));
}
}
private Optional<Account> dynamoGet(String number) {
return accounts.get(number);
}
private Optional<Account> dynamoGet(UUID uuid) {
return accounts.get(uuid);
}
private boolean dynamoCreate(Account account) {
return accounts.create(account);
}
private void dynamoUpdate(Account account) {
accounts.update(account);
}
private void dynamoDelete(final Account account) {
accounts.delete(account.getUuid());
}
private void dynamoChangeNumber(final Account account, final String number) {
accounts.changeNumber(account, number);
}
}
@@ -43,7 +43,7 @@ public class ContactDiscoveryWriter extends AccountDatabaseCrawlerListener {
// It’s less than ideal, but crawler listeners currently must not call update()
// with the accounts from the chunk, because updates cause the account instance to become stale. Instead, they
// must get a new copy, which they are free to update.
accounts.getByAccountIdentifier(account.getUuid()).ifPresent(a -> accounts.update(a, NOOP_UPDATER));
accounts.get(account.getUuid()).ifPresent(a -> accounts.update(a, NOOP_UPDATER));
}
}
}
@@ -53,9 +53,6 @@ public class Device {
@JsonProperty
private SignedPreKey signedPreKey;
@JsonProperty("pniSignedPreKey")
private SignedPreKey phoneNumberIdentitySignedPreKey;
@JsonProperty
private long lastSeen;
@@ -218,14 +215,6 @@ public class Device {
this.signedPreKey = signedPreKey;
}
public SignedPreKey getPhoneNumberIdentitySignedPreKey() {
return phoneNumberIdentitySignedPreKey;
}
public void setPhoneNumberIdentitySignedPreKey(final SignedPreKey phoneNumberIdentitySignedPreKey) {
this.phoneNumberIdentitySignedPreKey = phoneNumberIdentitySignedPreKey;
}
public long getPushTimestamp() {
return pushTimestamp;
}
@@ -0,0 +1,64 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import java.io.IOException;
class Key {
private final byte[] userMessageQueue;
private final byte[] userMessageQueueMetadata;
private final byte[] userMessageQueuePersistInProgress;
private final String address;
private final long deviceId;
Key(String address, long deviceId) {
this.address = address;
this.deviceId = deviceId;
this.userMessageQueue = ("user_queue::" + address + "::" + deviceId).getBytes();
this.userMessageQueueMetadata = ("user_queue_metadata::" + address + "::" + deviceId).getBytes();
this.userMessageQueuePersistInProgress = ("user_queue_persisting::" + address + "::" + deviceId).getBytes();
}
String getAddress() {
return address;
}
long getDeviceId() {
return deviceId;
}
byte[] getUserMessageQueue() {
return userMessageQueue;
}
byte[] getUserMessageQueueMetadata() {
return userMessageQueueMetadata;
}
byte[] getUserMessageQueuePersistInProgress() {
return userMessageQueuePersistInProgress;
}
static byte[] getUserMessageQueueIndex() {
return "user_queue_index".getBytes();
}
static Key fromUserMessageQueue(byte[] userMessageQueue) throws IOException {
try {
String[] parts = new String(userMessageQueue).split("::");
if (parts.length != 3) {
throw new IOException("Malformed key: " + new String(userMessageQueue));
}
return new Key(parts[1], Long.parseLong(parts[2]));
} catch (NumberFormatException e) {
throw new IOException(e);
}
}
}
@@ -5,12 +5,15 @@
package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -29,9 +32,7 @@ import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
import software.amazon.awssdk.services.dynamodb.model.Select;
import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
public class Keys extends AbstractDynamoDbStore {
public class KeysDynamoDb extends AbstractDynamoDbStore {
private final String tableName;
@@ -39,29 +40,30 @@ public class Keys extends AbstractDynamoDbStore {
static final String KEY_DEVICE_ID_KEY_ID = "DK";
static final String KEY_PUBLIC_KEY = "P";
private static final Timer STORE_KEYS_TIMER = Metrics.timer(name(Keys.class, "storeKeys"));
private static final Timer TAKE_KEY_FOR_DEVICE_TIMER = Metrics.timer(name(Keys.class, "takeKeyForDevice"));
private static final Timer GET_KEY_COUNT_TIMER = Metrics.timer(name(Keys.class, "getKeyCount"));
private static final Timer DELETE_KEYS_FOR_DEVICE_TIMER = Metrics.timer(name(Keys.class, "deleteKeysForDevice"));
private static final Timer DELETE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(Keys.class, "deleteKeysForAccount"));
private static final DistributionSummary CONTESTED_KEY_DISTRIBUTION = Metrics.summary(name(Keys.class, "contestedKeys"));
private static final DistributionSummary KEY_COUNT_DISTRIBUTION = Metrics.summary(name(Keys.class, "keyCount"));
private static final Timer STORE_KEYS_TIMER = Metrics.timer(name(KeysDynamoDb.class, "storeKeys"));
private static final Timer TAKE_KEY_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForDevice"));
private static final Timer TAKE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "takeKeyForAccount"));
private static final Timer GET_KEY_COUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "getKeyCount"));
private static final Timer DELETE_KEYS_FOR_DEVICE_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForDevice"));
private static final Timer DELETE_KEYS_FOR_ACCOUNT_TIMER = Metrics.timer(name(KeysDynamoDb.class, "deleteKeysForAccount"));
private static final DistributionSummary CONTESTED_KEY_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "contestedKeys"));
private static final DistributionSummary KEY_COUNT_DISTRIBUTION = Metrics.summary(name(KeysDynamoDb.class, "keyCount"));
public Keys(final DynamoDbClient dynamoDB, final String tableName) {
public KeysDynamoDb(final DynamoDbClient dynamoDB, final String tableName) {
super(dynamoDB);
this.tableName = tableName;
}
public void store(final UUID identifier, final long deviceId, final List<PreKey> keys) {
public void store(final Account account, final long deviceId, final List<PreKey> keys) {
STORE_KEYS_TIMER.record(() -> {
delete(identifier, deviceId);
delete(account.getUuid(), deviceId);
writeInBatches(keys, batch -> {
List<WriteRequest> items = new ArrayList<>();
for (final PreKey preKey : batch) {
items.add(WriteRequest.builder()
.putRequest(PutRequest.builder()
.item(getItemFromPreKey(identifier, deviceId, preKey))
.item(getItemFromPreKey(account.getUuid(), deviceId, preKey))
.build())
.build());
}
@@ -70,9 +72,9 @@ public class Keys extends AbstractDynamoDbStore {
});
}
public Optional<PreKey> take(final UUID identifier, final long deviceId) {
public Optional<PreKey> take(final Account account, final long deviceId) {
return TAKE_KEY_FOR_DEVICE_TIMER.record(() -> {
final AttributeValue partitionKey = getPartitionKey(identifier);
final AttributeValue partitionKey = getPartitionKey(account.getUuid());
QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.keyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
@@ -111,14 +113,26 @@ public class Keys extends AbstractDynamoDbStore {
});
}
public int getCount(final UUID identifier, final long deviceId) {
public Map<Long, PreKey> take(final Account account) {
return TAKE_KEYS_FOR_ACCOUNT_TIMER.record(() -> {
final Map<Long, PreKey> preKeysByDeviceId = new HashMap<>();
for (final Device device : account.getDevices()) {
take(account, device.getId()).ifPresent(preKey -> preKeysByDeviceId.put(device.getId(), preKey));
}
return preKeysByDeviceId;
});
}
public int getCount(final Account account, final long deviceId) {
return GET_KEY_COUNT_TIMER.record(() -> {
QueryRequest queryRequest = QueryRequest.builder()
.tableName(tableName)
.keyConditionExpression("#uuid = :uuid AND begins_with (#sort, :sortprefix)")
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID, "#sort", KEY_DEVICE_ID_KEY_ID))
.expressionAttributeValues(Map.of(
":uuid", getPartitionKey(identifier),
":uuid", getPartitionKey(account.getUuid()),
":sortprefix", getSortKeyPrefix(deviceId)))
.select(Select.COUNT)
.consistentRead(false)
@@ -146,7 +146,7 @@ public class MessagePersister implements Managed {
@VisibleForTesting
void persistQueue(final UUID accountUuid, final long deviceId) {
final Optional<Account> maybeAccount = accountsManager.getByAccountIdentifier(accountUuid);
final Optional<Account> maybeAccount = accountsManager.get(accountUuid);
if (maybeAccount.isEmpty()) {
logger.error("No account record found for account {}", accountUuid);
@@ -167,7 +167,7 @@ public class MessagesCache extends RedisClusterPubSubAdapter<String, String> imp
for (final byte[] bytes : serialized) {
try {
removedMessages.add(constructEntityFromEnvelope(MessageProtos.Envelope.parseFrom(bytes)));
removedMessages.add(constructEntityFromEnvelope(0, MessageProtos.Envelope.parseFrom(bytes)));
} catch (final InvalidProtocolBufferException e) {
logger.warn("Failed to parse envelope", e);
}
@@ -208,7 +208,7 @@ public class MessagesCache extends RedisClusterPubSubAdapter<String, String> imp
final long id = Long.parseLong(new String(queueItems.get(i + 1), StandardCharsets.UTF_8));
messageEntities.add(constructEntityFromEnvelope(message));
messageEntities.add(constructEntityFromEnvelope(id, message));
} catch (InvalidProtocolBufferException e) {
logger.warn("Failed to parse envelope", e);
}
@@ -376,8 +376,8 @@ public class MessagesCache extends RedisClusterPubSubAdapter<String, String> imp
}
@VisibleForTesting
static OutgoingMessageEntity constructEntityFromEnvelope(MessageProtos.Envelope envelope) {
return new OutgoingMessageEntity(
static OutgoingMessageEntity constructEntityFromEnvelope(long id, MessageProtos.Envelope envelope) {
return new OutgoingMessageEntity(id, true,
envelope.hasServerGuid() ? UUID.fromString(envelope.getServerGuid()) : null,
envelope.getType().getNumber(),
envelope.getRelay(),
@@ -385,7 +385,6 @@ public class MessagesCache extends RedisClusterPubSubAdapter<String, String> imp
envelope.getSource(),
envelope.hasSourceUuid() ? UUID.fromString(envelope.getSourceUuid()) : null,
envelope.getSourceDevice(),
envelope.hasDestinationUuid() ? UUID.fromString(envelope.getDestinationUuid()) : null,
envelope.hasLegacyMessage() ? envelope.getLegacyMessage().toByteArray() : null,
envelope.hasContent() ? envelope.getContent().toByteArray() : null,
envelope.hasServerTimestamp() ? envelope.getServerTimestamp() : 0);
@@ -46,7 +46,6 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
private static final String KEY_SOURCE = "SN";
private static final String KEY_SOURCE_UUID = "SU";
private static final String KEY_SOURCE_DEVICE = "SD";
private static final String KEY_DESTINATION_UUID = "DU";
private static final String KEY_MESSAGE = "M";
private static final String KEY_CONTENT = "C";
private static final String KEY_TTL = "E";
@@ -87,12 +86,6 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
.put(KEY_TYPE, AttributeValues.fromInt(message.getType().getNumber()))
.put(KEY_TIMESTAMP, AttributeValues.fromLong(message.getTimestamp()))
.put(KEY_TTL, AttributeValues.fromLong(getTtlForMessage(message)));
// TODO All messages should have a destination UUID by 2021-12-03, and this can be set unconditionally
if (message.hasDestinationUuid()) {
item.put(KEY_DESTINATION_UUID, AttributeValues.fromUUID(UUID.fromString(message.getDestinationUuid())));
}
if (message.hasRelay() && message.getRelay().length() > 0) {
item.put(KEY_RELAY, AttributeValues.fromString(message.getRelay()));
}
@@ -231,10 +224,9 @@ public class MessagesDynamoDb extends AbstractDynamoDbStore {
final String source = AttributeValues.getString(message, KEY_SOURCE, null);
final UUID sourceUuid = AttributeValues.getUUID(message, KEY_SOURCE_UUID, null);
final int sourceDevice = AttributeValues.getInt(message, KEY_SOURCE_DEVICE, 0);
final UUID destinationUuid = AttributeValues.getUUID(message, KEY_DESTINATION_UUID, null);
final byte[] messageBytes = AttributeValues.getByteArray(message, KEY_MESSAGE, null);
final byte[] content = AttributeValues.getByteArray(message, KEY_CONTENT, null);
return new OutgoingMessageEntity(messageUuid, type, relay, timestamp, source, sourceUuid, sourceDevice, destinationUuid, messageBytes, content, sortKey.getServerTimestamp());
return new OutgoingMessageEntity(-1L, false, messageUuid, type, relay, timestamp, source, sourceUuid, sourceDevice, messageBytes, content, sortKey.getServerTimestamp());
}
private void deleteRowsMatchingQuery(AttributeValue partitionKey, QueryRequest querySpec) {
@@ -62,7 +62,7 @@ public class NonNormalizedAccountCrawlerListener extends AccountDatabaseCrawlerL
workingNonNormalizedNumbers++;
try {
final Optional<Account> maybeConflictingAccount = accountsManager.getByE164(getNormalizedNumber(account));
final Optional<Account> maybeConflictingAccount = accountsManager.get(getNormalizedNumber(account));
if (maybeConflictingAccount.isPresent()) {
workingConflictingNumbers++;
@@ -1,85 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
import software.amazon.awssdk.services.dynamodb.model.ReturnValue;
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
import software.amazon.awssdk.services.dynamodb.model.UpdateItemResponse;
import java.util.Map;
import java.util.UUID;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
/**
* Manages a global, persistent mapping of phone numbers to phone number identifiers regardless of whether those
* numbers/identifiers are actually associated with an account.
*/
public class PhoneNumberIdentifiers {
private final DynamoDbClient dynamoDbClient;
private final String tableName;
@VisibleForTesting
static final String KEY_E164 = "P";
private static final String ATTR_PHONE_NUMBER_IDENTIFIER = "PNI";
private static final Timer GET_PNI_TIMER = Metrics.timer(name(PhoneNumberIdentifiers.class, "get"));
private static final Timer SET_PNI_TIMER = Metrics.timer(name(PhoneNumberIdentifiers.class, "set"));
public PhoneNumberIdentifiers(final DynamoDbClient dynamoDbClient, final String tableName) {
this.dynamoDbClient = dynamoDbClient;
this.tableName = tableName;
}
/**
* Returns the phone number identifier (PNI) associated with the given phone number.
*
* @param phoneNumber the phone number for which to retrieve a phone number identifier
* @return the phone number identifier associated with the given phone number
*/
public UUID getPhoneNumberIdentifier(final String phoneNumber) {
final GetItemResponse response = GET_PNI_TIMER.record(() -> dynamoDbClient.getItem(GetItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_E164, AttributeValues.fromString(phoneNumber)))
.projectionExpression(ATTR_PHONE_NUMBER_IDENTIFIER)
.build()));
final UUID phoneNumberIdentifier;
if (response.hasItem()) {
phoneNumberIdentifier = AttributeValues.getUUID(response.item(), ATTR_PHONE_NUMBER_IDENTIFIER, null);
} else {
phoneNumberIdentifier = generatePhoneNumberIdentifierIfNotExists(phoneNumber);
}
if (phoneNumberIdentifier == null) {
throw new RuntimeException("Could not retrieve phone number identifier from stored item");
}
return phoneNumberIdentifier;
}
@VisibleForTesting
UUID generatePhoneNumberIdentifierIfNotExists(final String phoneNumber) {
final UpdateItemResponse response = SET_PNI_TIMER.record(() -> dynamoDbClient.updateItem(UpdateItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_E164, AttributeValues.fromString(phoneNumber)))
.updateExpression("SET #pni = if_not_exists(#pni, :pni)")
.expressionAttributeNames(Map.of("#pni", ATTR_PHONE_NUMBER_IDENTIFIER))
.expressionAttributeValues(Map.of(":pni", AttributeValues.fromUUID(UUID.randomUUID())))
.returnValues(ReturnValue.ALL_NEW)
.build()));
return AttributeValues.getUUID(response.attributes(), ATTR_PHONE_NUMBER_IDENTIFIER, null);
}
}
@@ -1,235 +1,98 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import static com.codahale.metrics.MetricRegistry.name;
import com.google.common.annotations.VisibleForTesting;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemRequest;
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
import software.amazon.awssdk.services.dynamodb.model.QueryRequest;
import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest;
import software.amazon.awssdk.services.dynamodb.paginators.QueryIterable;
import org.whispersystems.textsecuregcm.storage.mappers.VersionedProfileMapper;
import org.whispersystems.textsecuregcm.util.Constants;
public class Profiles {
private final DynamoDbClient dynamoDbClient;
private final DynamoDbAsyncClient dynamoDbAsyncClient;
private final String tableName;
public static final String ID = "id";
public static final String UID = "uuid";
public static final String VERSION = "version";
public static final String NAME = "name";
public static final String AVATAR = "avatar";
public static final String ABOUT_EMOJI = "about_emoji";
public static final String ABOUT = "about";
public static final String PAYMENT_ADDRESS = "payment_address";
public static final String COMMITMENT = "commitment";
// UUID of the account that owns this profile; byte array
@VisibleForTesting
static final String KEY_ACCOUNT_UUID = "U";
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
// Version of this profile; string
@VisibleForTesting
static final String ATTR_VERSION = "V";
private final Timer setTimer = metricRegistry.timer(name(Profiles.class, "set" ));
private final Timer getTimer = metricRegistry.timer(name(Profiles.class, "get" ));
private final Timer deleteTimer = metricRegistry.timer(name(Profiles.class, "delete"));
// User's name; string
private static final String ATTR_NAME = "N";
private final FaultTolerantDatabase database;
// Avatar path/filename; string
private static final String ATTR_AVATAR = "A";
// Bio/about text; string
private static final String ATTR_ABOUT = "B";
// Bio/about emoji; string
private static final String ATTR_EMOJI = "E";
// Payment address; string
private static final String ATTR_PAYMENT_ADDRESS = "P";
// Commitment; byte array
private static final String ATTR_COMMITMENT = "C";
private static final Map<String, String> UPDATE_EXPRESSION_ATTRIBUTE_NAMES = Map.of(
"#commitment", ATTR_COMMITMENT,
"#name", ATTR_NAME,
"#avatar", ATTR_AVATAR,
"#about", ATTR_ABOUT,
"#aboutEmoji", ATTR_EMOJI,
"#paymentAddress", ATTR_PAYMENT_ADDRESS);
private static final Timer SET_PROFILES_TIMER = Metrics.timer(name(Profiles.class, "set"));
private static final Timer GET_PROFILE_TIMER = Metrics.timer(name(Profiles.class, "get"));
private static final Timer DELETE_PROFILES_TIMER = Metrics.timer(name(Profiles.class, "delete"));
public Profiles(final DynamoDbClient dynamoDbClient,
final DynamoDbAsyncClient dynamoDbAsyncClient,
final String tableName) {
this.dynamoDbClient = dynamoDbClient;
this.dynamoDbAsyncClient = dynamoDbAsyncClient;
this.tableName = tableName;
public Profiles(FaultTolerantDatabase database) {
this.database = database;
this.database.getDatabase().registerRowMapper(new VersionedProfileMapper());
}
public void set(final UUID uuid, final VersionedProfile profile) {
SET_PROFILES_TIMER.record(() -> {
dynamoDbClient.updateItem(UpdateItemRequest.builder()
.tableName(tableName)
.key(buildPrimaryKey(uuid, profile.getVersion()))
.updateExpression(buildUpdateExpression(profile))
.expressionAttributeNames(UPDATE_EXPRESSION_ATTRIBUTE_NAMES)
.expressionAttributeValues(buildUpdateExpressionAttributeValues(profile))
.build());
});
public void set(UUID uuid, VersionedProfile profile) {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = setTimer.time()) {
handle.createUpdate(
"INSERT INTO profiles ("
+ UID + ", "
+ VERSION + ", "
+ NAME + ", "
+ AVATAR + ", "
+ ABOUT_EMOJI + ", "
+ ABOUT + ", "
+ PAYMENT_ADDRESS + ", "
+ COMMITMENT + ") "
+ "VALUES (:uuid, :version, :name, :avatar, :about_emoji, :about, :payment_address, :commitment) "
+ "ON CONFLICT (" + UID + ", " + VERSION + ") "
+ "DO UPDATE SET "
+ NAME + " = EXCLUDED." + NAME + ", "
+ AVATAR + " = EXCLUDED." + AVATAR + ", "
+ ABOUT + " = EXCLUDED." + ABOUT + ", "
+ ABOUT_EMOJI + " = EXCLUDED." + ABOUT_EMOJI + ", "
+ PAYMENT_ADDRESS + " = EXCLUDED." + PAYMENT_ADDRESS)
.bind("uuid", uuid)
.bind("version", profile.getVersion())
.bind("name", profile.getName())
.bind("avatar", profile.getAvatar())
.bind("about_emoji", profile.getAboutEmoji())
.bind("about", profile.getAbout())
.bind("payment_address", profile.getPaymentAddress())
.bind("commitment", profile.getCommitment())
.execute();
}
}));
}
private static Map<String, AttributeValue> buildPrimaryKey(final UUID uuid, final String version) {
return Map.of(
KEY_ACCOUNT_UUID, AttributeValues.fromUUID(uuid),
ATTR_VERSION, AttributeValues.fromString(version));
public Optional<VersionedProfile> get(UUID uuid, String version) {
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = getTimer.time()) {
return handle.createQuery("SELECT * FROM profiles WHERE " + UID + " = :uuid AND " + VERSION + " = :version")
.bind("uuid", uuid)
.bind("version", version)
.mapTo(VersionedProfile.class)
.findFirst();
}
}));
}
@VisibleForTesting
static String buildUpdateExpression(final VersionedProfile profile) {
final List<String> updatedAttributes = new ArrayList<>(5);
final List<String> deletedAttributes = new ArrayList<>(5);
if (StringUtils.isNotBlank(profile.getName())) {
updatedAttributes.add("name");
} else {
deletedAttributes.add("name");
}
if (StringUtils.isNotBlank(profile.getAvatar())) {
updatedAttributes.add("avatar");
} else {
deletedAttributes.add("avatar");
}
if (StringUtils.isNotBlank(profile.getAbout())) {
updatedAttributes.add("about");
} else {
deletedAttributes.add("about");
}
if (StringUtils.isNotBlank(profile.getAboutEmoji())) {
updatedAttributes.add("aboutEmoji");
} else {
deletedAttributes.add("aboutEmoji");
}
if (StringUtils.isNotBlank(profile.getPaymentAddress())) {
updatedAttributes.add("paymentAddress");
} else {
deletedAttributes.add("paymentAddress");
}
final StringBuilder updateExpressionBuilder = new StringBuilder(
"SET #commitment = if_not_exists(#commitment, :commitment)");
if (!updatedAttributes.isEmpty()) {
updatedAttributes.forEach(token -> updateExpressionBuilder
.append(", #")
.append(token)
.append(" = :")
.append(token));
}
if (!deletedAttributes.isEmpty()) {
updateExpressionBuilder.append(" REMOVE ");
updateExpressionBuilder.append(deletedAttributes.stream()
.map(token -> "#" + token)
.collect(Collectors.joining(", ")));
}
return updateExpressionBuilder.toString();
}
@VisibleForTesting
static Map<String, AttributeValue> buildUpdateExpressionAttributeValues(final VersionedProfile profile) {
final Map<String, AttributeValue> expressionValues = new HashMap<>();
expressionValues.put(":commitment", AttributeValues.fromByteArray(profile.getCommitment()));
if (StringUtils.isNotBlank(profile.getName())) {
expressionValues.put(":name", AttributeValues.fromString(profile.getName()));
}
if (StringUtils.isNotBlank(profile.getAvatar())) {
expressionValues.put(":avatar", AttributeValues.fromString(profile.getAvatar()));
}
if (StringUtils.isNotBlank(profile.getAbout())) {
expressionValues.put(":about", AttributeValues.fromString(profile.getAbout()));
}
if (StringUtils.isNotBlank(profile.getAboutEmoji())) {
expressionValues.put(":aboutEmoji", AttributeValues.fromString(profile.getAboutEmoji()));
}
if (StringUtils.isNotBlank(profile.getPaymentAddress())) {
expressionValues.put(":paymentAddress", AttributeValues.fromString(profile.getPaymentAddress()));
}
return expressionValues;
}
public Optional<VersionedProfile> get(final UUID uuid, final String version) {
return GET_PROFILE_TIMER.record(() -> {
final GetItemResponse response = dynamoDbClient.getItem(GetItemRequest.builder()
.tableName(tableName)
.key(buildPrimaryKey(uuid, version))
.consistentRead(true)
.build());
return response.hasItem() ? Optional.of(fromItem(response.item())) : Optional.empty();
});
}
private static VersionedProfile fromItem(final Map<String, AttributeValue> item) {
return new VersionedProfile(
AttributeValues.getString(item, ATTR_VERSION, null),
AttributeValues.getString(item, ATTR_NAME, null),
AttributeValues.getString(item, ATTR_AVATAR, null),
AttributeValues.getString(item, ATTR_EMOJI, null),
AttributeValues.getString(item, ATTR_ABOUT, null),
AttributeValues.getString(item, ATTR_PAYMENT_ADDRESS, null),
AttributeValues.getByteArray(item, ATTR_COMMITMENT, null));
}
public void deleteAll(final UUID uuid) {
DELETE_PROFILES_TIMER.record(() -> {
final AttributeValue uuidAttributeValue = AttributeValues.fromUUID(uuid);
final QueryIterable queryIterable = dynamoDbClient.queryPaginator(QueryRequest.builder()
.tableName(tableName)
.keyConditionExpression("#uuid = :uuid")
.expressionAttributeNames(Map.of("#uuid", KEY_ACCOUNT_UUID))
.expressionAttributeValues(Map.of(":uuid", uuidAttributeValue))
.projectionExpression(ATTR_VERSION)
.consistentRead(true)
.build());
CompletableFuture.allOf(queryIterable.items().stream()
.map(item -> dynamoDbAsyncClient.deleteItem(DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(
KEY_ACCOUNT_UUID, uuidAttributeValue,
ATTR_VERSION, item.get(ATTR_VERSION)))
.build()))
.toArray(CompletableFuture[]::new)).join();
});
public void deleteAll(UUID uuid) {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = deleteTimer.time()) {
handle.createUpdate("DELETE FROM profiles WHERE " + UID + " = :uuid")
.bind("uuid", uuid)
.execute();
}
}));
}
}
@@ -8,29 +8,29 @@ package org.whispersystems.textsecuregcm.storage;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.lettuce.core.RedisException;
import java.io.IOException;
import java.util.Optional;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.util.SystemMapper;
import java.io.IOException;
import java.util.Optional;
import java.util.UUID;
public class ProfilesManager {
private final Logger logger = LoggerFactory.getLogger(ProfilesManager.class);
private static final String CACHE_PREFIX = "profiles::";
private final Profiles profiles;
private final Profiles profiles;
private final FaultTolerantRedisCluster cacheCluster;
private final ObjectMapper mapper;
private final ObjectMapper mapper;
public ProfilesManager(final Profiles profiles,
final FaultTolerantRedisCluster cacheCluster) {
this.profiles = profiles;
this.cacheCluster = cacheCluster;
this.mapper = SystemMapper.getMapper();
public ProfilesManager(Profiles profiles, FaultTolerantRedisCluster cacheCluster) {
this.profiles = profiles;
this.cacheCluster = cacheCluster;
this.mapper = SystemMapper.getMapper();
}
public void set(UUID uuid, VersionedProfile versionedProfile) {
@@ -46,7 +46,7 @@ public class ProfilesManager {
public Optional<VersionedProfile> get(UUID uuid, String version) {
Optional<VersionedProfile> profile = memcacheGet(uuid, version);
if (profile.isEmpty()) {
if (!profile.isPresent()) {
profile = profiles.get(uuid, version);
profile.ifPresent(versionedProfile -> memcacheSet(uuid, versionedProfile));
}
@@ -58,7 +58,7 @@ public class PushFeedbackProcessor extends AccountDatabaseCrawlerListener {
if (update) {
// fetch a new version, since the chunk is shared and implicitly read-only
accountsManager.getByAccountIdentifier(account.getUuid()).ifPresent(accountToUpdate -> {
accountsManager.get(account.getUuid()).ifPresent(accountToUpdate -> {
accountsManager.update(accountToUpdate, a -> {
for (Device device : a.getDevices()) {
if (deviceNeedsUpdate(device)) {
@@ -24,7 +24,7 @@ public class RefreshingAccountAndDeviceSupplier implements Supplier<Pair<Account
@Override
public Pair<Account, Device> get() {
if (account.isStale()) {
account = accountsManager.getByAccountIdentifier(account.getUuid())
account = accountsManager.get(account.getUuid())
.orElseThrow(() -> new RuntimeException("Could not find account"));
device = account.getDevice(device.getId())
.orElseThrow(() -> new RefreshingAccountAndDeviceNotFoundException("Could not find device"));
@@ -1,114 +1,75 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import org.whispersystems.textsecuregcm.util.UUIDUtil;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import java.util.Collections;
import java.util.HashMap;
import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import org.whispersystems.textsecuregcm.storage.mappers.RemoteConfigRowMapper;
import org.whispersystems.textsecuregcm.util.Constants;
public class RemoteConfigs {
private final DynamoDbClient dynamoDbClient;
private final String tableName;
public static final String ID = "id";
public static final String NAME = "name";
public static final String PERCENTAGE = "percentage";
public static final String UUIDS = "uuids";
public static final String DEFAULT_VALUE = "default_value";
public static final String VALUE = "value";
public static final String HASH_KEY = "hash_key";
// Config name; string
static final String KEY_NAME = "N";
// Rollout percentage; integer
private static final String ATTR_PERCENTAGE = "P";
// Enrolled UUIDs (ACIs); list of byte arrays
private static final String ATTR_UUIDS = "U";
// Default value; string
private static final String ATTR_DEFAULT_VALUE = "D";
// Value when enrolled; string
private static final String ATTR_VALUE = "V";
// Hash key; string
private static final String ATTR_HASH_KEY = "H";
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Timer setTimer = metricRegistry.timer(name(RemoteConfigs.class, "set" ));
private final Timer getAllTimer = metricRegistry.timer(name(RemoteConfigs.class, "getAll"));
private final Timer deleteTimer = metricRegistry.timer(name(RemoteConfigs.class, "delete"));
public RemoteConfigs(final DynamoDbClient dynamoDbClient, final String tableName) {
this.dynamoDbClient = dynamoDbClient;
this.tableName = tableName;
private final FaultTolerantDatabase database;
public RemoteConfigs(FaultTolerantDatabase database) {
this.database = database;
this.database.getDatabase().registerRowMapper(new RemoteConfigRowMapper());
this.database.getDatabase().registerArrayType(UUID.class, "uuid");
}
public void set(final RemoteConfig remoteConfig) {
final Map<String, AttributeValue> item = new HashMap<>(Map.of(
KEY_NAME, AttributeValues.fromString(remoteConfig.getName()),
ATTR_PERCENTAGE, AttributeValues.fromInt(remoteConfig.getPercentage())));
if (remoteConfig.getUuids() != null && !remoteConfig.getUuids().isEmpty()) {
final List<SdkBytes> uuidByteSets = remoteConfig.getUuids().stream()
.map(UUIDUtil::toByteBuffer)
.map(SdkBytes::fromByteBuffer)
.collect(Collectors.toList());
item.put(ATTR_UUIDS, AttributeValue.builder().bs(uuidByteSets).build());
}
if (remoteConfig.getDefaultValue() != null) {
item.put(ATTR_DEFAULT_VALUE, AttributeValues.fromString(remoteConfig.getDefaultValue()));
}
if (remoteConfig.getValue() != null) {
item.put(ATTR_VALUE, AttributeValues.fromString(remoteConfig.getValue()));
}
if (remoteConfig.getHashKey() != null) {
item.put(ATTR_HASH_KEY, AttributeValues.fromString(remoteConfig.getHashKey()));
}
dynamoDbClient.putItem(PutItemRequest.builder()
.tableName(tableName)
.item(item)
.build());
public void set(RemoteConfig remoteConfig) {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = setTimer.time()) {
handle.createUpdate("INSERT INTO remote_config (" + NAME + ", " + PERCENTAGE + ", " + UUIDS + ", " + DEFAULT_VALUE + ", " + VALUE + ", " + HASH_KEY + ") VALUES (:name, :percentage, :uuids, :default_value, :value, :hash_key) ON CONFLICT(" + NAME + ") DO UPDATE SET " + PERCENTAGE + " = EXCLUDED." + PERCENTAGE + ", " + UUIDS + " = EXCLUDED." + UUIDS + ", " + DEFAULT_VALUE + " = EXCLUDED." + DEFAULT_VALUE + ", " + VALUE + " = EXCLUDED." + VALUE + ", " + HASH_KEY + " = EXCLUDED." + HASH_KEY)
.bind("name", remoteConfig.getName())
.bind("percentage", remoteConfig.getPercentage())
.bind("uuids", remoteConfig.getUuids().toArray(new UUID[0]))
.bind("default_value", remoteConfig.getDefaultValue())
.bind("value", remoteConfig.getValue())
.bind("hash_key", remoteConfig.getHashKey())
.execute();
}
}));
}
public List<RemoteConfig> getAll() {
return dynamoDbClient.scanPaginator(ScanRequest.builder()
.tableName(tableName)
.consistentRead(true)
.build())
.items()
.stream()
.map(item -> {
final String name = AttributeValues.getString(item, KEY_NAME, null);
final int percentage = AttributeValues.getInt(item, ATTR_PERCENTAGE, 0);
final String defaultValue = AttributeValues.getString(item, ATTR_DEFAULT_VALUE, null);
final String value = AttributeValues.getString(item, ATTR_VALUE, null);
final String hashKey = AttributeValues.getString(item, ATTR_HASH_KEY, null);
final Set<UUID> uuids;
if (item.containsKey(ATTR_UUIDS)) {
uuids = item.get(ATTR_UUIDS).bs().stream()
.map(sdkBytes -> UUIDUtil.fromByteBuffer(sdkBytes.asByteBuffer()))
.collect(Collectors.toSet());
} else {
uuids = Collections.emptySet();
}
return new RemoteConfig(name, percentage, uuids, defaultValue, value, hashKey);
})
.collect(Collectors.toList());
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = getAllTimer.time()) {
return handle.createQuery("SELECT * FROM remote_config")
.mapTo(RemoteConfig.class)
.list();
}
}));
}
public void delete(final String name) {
dynamoDbClient.deleteItem(DeleteItemRequest.builder()
.tableName(tableName)
.key(Map.of(KEY_NAME, AttributeValues.fromString(name)))
.build());
public void delete(String name) {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = deleteTimer.time()) {
handle.createUpdate("DELETE FROM remote_config WHERE " + NAME + " = :name")
.bind("name", name)
.execute();
}
}));
}
}
@@ -5,26 +5,70 @@
package org.whispersystems.textsecuregcm.storage;
import com.google.common.base.Suppliers;
import com.google.common.annotations.VisibleForTesting;
import io.dropwizard.lifecycle.Managed;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.Util;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import java.util.concurrent.atomic.AtomicReference;
public class RemoteConfigsManager {
public class RemoteConfigsManager implements Managed {
private final Logger logger = LoggerFactory.getLogger(RemoteConfigsManager.class);
private final RemoteConfigs remoteConfigs;
private final long sleepInterval;
private final Supplier<List<RemoteConfig>> remoteConfigSupplier;
private final AtomicReference<List<RemoteConfig>> cachedConfigs = new AtomicReference<>(new LinkedList<>());
public RemoteConfigsManager(RemoteConfigs remoteConfigs) {
this.remoteConfigs = remoteConfigs;
this(remoteConfigs, TimeUnit.SECONDS.toMillis(10));
}
remoteConfigSupplier =
Suppliers.memoizeWithExpiration(remoteConfigs::getAll, 10, TimeUnit.SECONDS);
@VisibleForTesting
public RemoteConfigsManager(RemoteConfigs remoteConfigs, long sleepInterval) {
this.remoteConfigs = remoteConfigs;
this.sleepInterval = sleepInterval;
}
@Override
public void start() {
refreshCache();
new Thread(() -> {
while (true) {
try {
refreshCache();
} catch (Throwable t) {
logger.warn("Error updating remote configs cache", t);
}
Util.sleep(sleepInterval);
}
}).start();
}
private void refreshCache() {
this.cachedConfigs.set(remoteConfigs.getAll());
synchronized (this.cachedConfigs) {
this.cachedConfigs.notifyAll();
}
}
@VisibleForTesting
void waitForCacheRefresh() throws InterruptedException {
synchronized (this.cachedConfigs) {
this.cachedConfigs.wait();
}
}
public List<RemoteConfig> getAll() {
return remoteConfigSupplier.get();
return cachedConfigs.get();
}
public void set(RemoteConfig config) {
@@ -34,4 +78,9 @@ public class RemoteConfigsManager {
public void delete(String name) {
remoteConfigs.delete(name);
}
@Override
public void stop() throws Exception {
}
}
@@ -54,7 +54,7 @@ public class ReportMessageManager {
}
}
public void report(String sourceNumber, UUID messageGuid, UUID reporterUuid) {
public void report(String sourceNumber, UUID messageGuid) {
final boolean found = reportMessageDynamoDb.remove(hash(messageGuid, sourceNumber));
@@ -62,7 +62,7 @@ public class ReportMessageManager {
rateLimitCluster.useCluster(connection -> {
final String reportedSenderKey = getReportedSenderKey(sourceNumber);
connection.sync().pfadd(reportedSenderKey, reporterUuid.toString());
connection.sync().pfadd(reportedSenderKey, sourceNumber);
connection.sync().expire(reportedSenderKey, counterTtl.toSeconds());
});
@@ -1,91 +1,58 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static org.whispersystems.textsecuregcm.metrics.MetricsUtil.name;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import java.util.Map;
import org.whispersystems.textsecuregcm.util.Constants;
import java.util.Optional;
import java.util.UUID;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.AttributeValues;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanRequest;
import software.amazon.awssdk.services.dynamodb.model.ScanResponse;
import software.amazon.awssdk.services.dynamodb.paginators.ScanIterable;
import static com.codahale.metrics.MetricRegistry.name;
public class ReservedUsernames {
private final DynamoDbClient dynamoDbClient;
private final String tableName;
public static final String ID = "id";
public static final String UID = "uuid";
public static final String USERNAME = "username";
private final LoadingCache<String, Pattern> patternCache = CacheBuilder.newBuilder()
.maximumSize(1_000)
.build(new CacheLoader<>() {
@Override
public Pattern load(final String s) {
return Pattern.compile(s, Pattern.CASE_INSENSITIVE);
}
});
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Timer queryTimer = metricRegistry.timer(name(ReservedUsernames.class, "query"));
private final FaultTolerantDatabase database;
public ReservedUsernames(FaultTolerantDatabase database) {
this.database = database;
}
public boolean isReserved(String username, UUID uuid) {
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = queryTimer.time()) {
Optional<Integer> reservations = handle.createQuery("SELECT COUNT(*) FROM reserved_usernames WHERE " + UID + " != :uuid AND :username ~* " + USERNAME)
.bind("username", username)
.bind("uuid", uuid)
.mapTo(Integer.class)
.findFirst();
return reservations.isPresent() && reservations.get() > 0;
}
}));
}
@VisibleForTesting
static final String KEY_PATTERN = "P";
private static final String ATTR_RESERVED_FOR_UUID = "U";
private static final Timer IS_RESERVED_TIMER = Metrics.timer(name(ReservedUsernames.class, "isReserved"));
private static final Logger log = LoggerFactory.getLogger(ReservedUsernames.class);
public ReservedUsernames(final DynamoDbClient dynamoDbClient, final String tableName) {
this.dynamoDbClient = dynamoDbClient;
this.tableName = tableName;
public void setReserved(String username, UUID reservedFor) {
database.use(jdbi -> jdbi.useHandle(handle -> {
handle.createUpdate("INSERT INTO reserved_usernames (" + USERNAME + ", " + UID + ") VALUES(:username, :uuid)")
.bind("username", username)
.bind("uuid", reservedFor)
.execute();
}));
}
public boolean isReserved(final String username, final UUID accountIdentifier) {
return IS_RESERVED_TIMER.record(() -> {
final ScanIterable scanIterable = dynamoDbClient.scanPaginator(ScanRequest.builder()
.tableName(tableName)
.build());
for (final ScanResponse scanResponse : scanIterable) {
if (scanResponse.hasItems()) {
for (final Map<String, AttributeValue> item : scanResponse.items()) {
try {
final Pattern pattern = patternCache.get(item.get(KEY_PATTERN).s());
final UUID reservedFor = AttributeValues.getUUID(item, ATTR_RESERVED_FOR_UUID, null);
if (pattern.matcher(username).matches() && !accountIdentifier.equals(reservedFor)) {
return true;
}
} catch (final Exception e) {
log.error("Failed to load pattern from item: {}", item, e);
}
}
}
}
return false;
});
}
public void reserveUsername(final String pattern, final UUID reservedFor) {
dynamoDbClient.putItem(PutItemRequest.builder()
.tableName(tableName)
.item(Map.of(
KEY_PATTERN, AttributeValues.fromString(pattern),
ATTR_RESERVED_FOR_UUID, AttributeValues.fromUUID(reservedFor)))
.build());
}
}
@@ -1,9 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
public class UsernameNotAvailableException extends Exception {
}
@@ -0,0 +1,90 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import static com.codahale.metrics.MetricRegistry.name;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import java.sql.SQLException;
import java.util.Optional;
import java.util.UUID;
import org.jdbi.v3.core.JdbiException;
import org.whispersystems.textsecuregcm.util.Constants;
public class Usernames {
public static final String ID = "id";
public static final String UID = "uuid";
public static final String USERNAME = "username";
private final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private final Timer createTimer = metricRegistry.timer(name(Usernames.class, "create" ));
private final Timer deleteTimer = metricRegistry.timer(name(Usernames.class, "delete" ));
private final Timer getByUsernameTimer = metricRegistry.timer(name(Usernames.class, "getByUsername"));
private final Timer getByUuidTimer = metricRegistry.timer(name(Usernames.class, "getByUuid" ));
private final FaultTolerantDatabase database;
public Usernames(FaultTolerantDatabase database) {
this.database = database;
}
public boolean put(UUID uuid, String username) {
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = createTimer.time()) {
int modified = handle.createUpdate("INSERT INTO usernames (" + UID + ", " + USERNAME + ") VALUES (:uuid, :username) ON CONFLICT (" + UID + ") DO UPDATE SET " + USERNAME + " = EXCLUDED.username")
.bind("uuid", uuid)
.bind("username", username)
.execute();
return modified > 0;
} catch (JdbiException e) {
if (e.getCause() instanceof SQLException) {
if (((SQLException)e.getCause()).getSQLState().equals("23505")) {
return false;
}
}
throw e;
}
}));
}
public void delete(UUID uuid) {
database.use(jdbi -> jdbi.useHandle(handle -> {
try (Timer.Context ignored = deleteTimer.time()) {
handle.createUpdate("DELETE FROM usernames WHERE " + UID + " = :uuid")
.bind("uuid", uuid)
.execute();
}
}));
}
public Optional<UUID> get(String username) {
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = getByUsernameTimer.time()) {
return handle.createQuery("SELECT " + UID + " FROM usernames WHERE " + USERNAME + " = :username")
.bind("username", username)
.mapTo(UUID.class)
.findFirst();
}
}));
}
public Optional<String> get(UUID uuid) {
return database.with(jdbi -> jdbi.withHandle(handle -> {
try (Timer.Context ignored = getByUuidTimer.time()) {
return handle.createQuery("SELECT " + USERNAME + " FROM usernames WHERE " + UID + " = :uuid")
.bind("uuid", uuid)
.mapTo(String.class)
.findFirst();
}
}));
}
}
@@ -0,0 +1,181 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SharedMetricRegistries;
import com.codahale.metrics.Timer;
import io.lettuce.core.RedisException;
import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisCluster;
import org.whispersystems.textsecuregcm.util.Constants;
import java.util.Optional;
import java.util.UUID;
import static com.codahale.metrics.MetricRegistry.name;
public class UsernamesManager {
private static final MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate(Constants.METRICS_NAME);
private static final Timer createTimer = metricRegistry.timer(name(UsernamesManager.class, "create" ));
private static final Timer deleteTimer = metricRegistry.timer(name(UsernamesManager.class, "delete" ));
private static final Timer getByUuidTimer = metricRegistry.timer(name(UsernamesManager.class, "getByUuid" ));
private static final Timer getByUsernameTimer = metricRegistry.timer(name(UsernamesManager.class, "getByUsername" ));
private static final Timer redisSetTimer = metricRegistry.timer(name(UsernamesManager.class, "redisSet" ));
private static final Timer redisUuidGetTimer = metricRegistry.timer(name(UsernamesManager.class, "redisUuidGet" ));
private static final Timer redisUsernameGetTimer = metricRegistry.timer(name(UsernamesManager.class, "redisUsernameGet"));
private final Logger logger = LoggerFactory.getLogger(UsernamesManager.class);
private final Usernames usernames;
private final ReservedUsernames reservedUsernames;
private final FaultTolerantRedisCluster cacheCluster;
public UsernamesManager(Usernames usernames, ReservedUsernames reservedUsernames, FaultTolerantRedisCluster cacheCluster) {
this.usernames = usernames;
this.reservedUsernames = reservedUsernames;
this.cacheCluster = cacheCluster;
}
public boolean put(UUID uuid, String username) {
try (Timer.Context ignored = createTimer.time()) {
if (reservedUsernames.isReserved(username, uuid)) {
return false;
}
if (databasePut(uuid, username)) {
redisSet(uuid, username, true);
return true;
}
return false;
}
}
public Optional<UUID> get(String username) {
try (Timer.Context ignored = getByUsernameTimer.time()) {
Optional<UUID> uuid = redisGet(username);
if (uuid.isPresent()) {
return uuid;
}
Optional<UUID> retrieved = databaseGet(username);
retrieved.ifPresent(retrievedUuid -> redisSet(retrievedUuid, username, false));
return retrieved;
}
}
public Optional<String> get(UUID uuid) {
try (Timer.Context ignored = getByUuidTimer.time()) {
Optional<String> username = redisGet(uuid);
if (username.isPresent()) {
return username;
}
Optional<String> retrieved = databaseGet(uuid);
retrieved.ifPresent(retrievedUsername -> redisSet(uuid, retrievedUsername, false));
return retrieved;
}
}
public void delete(UUID uuid) {
try (Timer.Context ignored = deleteTimer.time()) {
redisDelete(uuid);
databaseDelete(uuid);
}
}
private boolean databasePut(UUID uuid, String username) {
return usernames.put(uuid, username);
}
private Optional<UUID> databaseGet(String username) {
return usernames.get(username);
}
private void databaseDelete(UUID uuid) {
usernames.delete(uuid);
}
private Optional<String> databaseGet(UUID uuid) {
return usernames.get(uuid);
}
private void redisSet(UUID uuid, String username, boolean required) {
final String uuidMapKey = getUuidMapKey(uuid);
final String usernameMapKey = getUsernameMapKey(username);
try (Timer.Context ignored = redisSetTimer.time()) {
cacheCluster.useCluster(connection -> {
final RedisAdvancedClusterCommands<String, String> commands = connection.sync();
final Optional<String> maybeOldUsername = Optional.ofNullable(commands.get(uuidMapKey));
maybeOldUsername.ifPresent(oldUsername -> commands.del(getUsernameMapKey(oldUsername)));
commands.set(uuidMapKey, username);
commands.set(usernameMapKey, uuid.toString());
});
} catch (RedisException e) {
if (required) throw e;
else logger.warn("Ignoring Redis failure", e);
}
}
private Optional<UUID> redisGet(String username) {
try (Timer.Context ignored = redisUsernameGetTimer.time()) {
final String result = cacheCluster.withCluster(connection -> connection.sync().get(getUsernameMapKey(username)));
if (result == null) return Optional.empty();
else return Optional.of(UUID.fromString(result));
} catch (RedisException e) {
logger.warn("Redis get failure", e);
return Optional.empty();
}
}
private Optional<String> redisGet(UUID uuid) {
try (Timer.Context ignored = redisUuidGetTimer.time()) {
final String result = cacheCluster.withCluster(connection -> connection.sync().get(getUuidMapKey(uuid)));
return Optional.ofNullable(result);
} catch (RedisException e) {
logger.warn("Redis get failure", e);
return Optional.empty();
}
}
private void redisDelete(UUID uuid) {
try (Timer.Context ignored = redisUuidGetTimer.time()) {
cacheCluster.useCluster(connection -> {
final RedisAdvancedClusterCommands<String, String> commands = connection.sync();
commands.del(getUuidMapKey(uuid));
redisGet(uuid).ifPresent(username -> {
commands.del(getUsernameMapKey(username));
});
});
}
}
private String getUuidMapKey(UUID uuid) {
return "UsernameByUuid::" + uuid.toString();
}
private String getUsernameMapKey(String username) {
return "UsernameByUsername::" + username;
}
}
@@ -5,36 +5,41 @@
package org.whispersystems.textsecuregcm.storage;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.whispersystems.textsecuregcm.util.ByteArrayAdapter;
import java.util.Arrays;
import java.util.Objects;
public class VersionedProfile {
private final String version;
private final String name;
private final String avatar;
private final String aboutEmoji;
private final String about;
private final String paymentAddress;
@JsonProperty
private String version;
@JsonProperty
private String name;
@JsonProperty
private String avatar;
@JsonProperty
private String aboutEmoji;
@JsonProperty
private String about;
@JsonProperty
private String paymentAddress;
@JsonProperty
@JsonSerialize(using = ByteArrayAdapter.Serializing.class)
@JsonDeserialize(using = ByteArrayAdapter.Deserializing.class)
private byte[] commitment;
@JsonCreator
public VersionedProfile() {}
public VersionedProfile(
@JsonProperty("version") final String version,
@JsonProperty("name") final String name,
@JsonProperty("avatar") final String avatar,
@JsonProperty("aboutEmoji") final String aboutEmoji,
@JsonProperty("about") final String about,
@JsonProperty("paymentAddress") final String paymentAddress,
@JsonProperty("commitment") final byte[] commitment) {
String version, String name, String avatar, String aboutEmoji, String about, String paymentAddress,
byte[] commitment) {
this.version = version;
this.name = name;
this.avatar = avatar;
@@ -71,23 +76,4 @@ public class VersionedProfile {
public byte[] getCommitment() {
return commitment;
}
@Override
public boolean equals(final Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
final VersionedProfile that = (VersionedProfile) o;
return Objects.equals(version, that.version) && Objects.equals(name, that.name) && Objects.equals(avatar,
that.avatar) && Objects.equals(aboutEmoji, that.aboutEmoji) && Objects.equals(about, that.about)
&& Objects.equals(paymentAddress, that.paymentAddress) && Arrays.equals(commitment, that.commitment);
}
@Override
public int hashCode() {
int result = Objects.hash(version, name, avatar, aboutEmoji, about, paymentAddress);
result = 31 * result + Arrays.hashCode(commitment);
return result;
}
}
@@ -0,0 +1,31 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage.mappers;
import org.jdbi.v3.core.mapper.RowMapper;
import org.jdbi.v3.core.statement.StatementContext;
import org.whispersystems.textsecuregcm.storage.RemoteConfig;
import org.whispersystems.textsecuregcm.storage.RemoteConfigs;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.UUID;
public class RemoteConfigRowMapper implements RowMapper<RemoteConfig> {
@Override
public RemoteConfig map(ResultSet rs, StatementContext ctx) throws SQLException {
return new RemoteConfig(rs.getString(RemoteConfigs.NAME),
rs.getInt(RemoteConfigs.PERCENTAGE),
new HashSet<>(Arrays.asList((UUID[])rs.getArray(RemoteConfigs.UUIDS).getArray())),
rs.getString(RemoteConfigs.DEFAULT_VALUE),
rs.getString(RemoteConfigs.VALUE),
rs.getString(RemoteConfigs.HASH_KEY));
}
}
@@ -0,0 +1,28 @@
/*
* Copyright 2013-2020 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.storage.mappers;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.jdbi.v3.core.mapper.RowMapper;
import org.jdbi.v3.core.statement.StatementContext;
import org.whispersystems.textsecuregcm.storage.Profiles;
import org.whispersystems.textsecuregcm.storage.VersionedProfile;
public class VersionedProfileMapper implements RowMapper<VersionedProfile> {
@Override
public VersionedProfile map(ResultSet resultSet, StatementContext ctx) throws SQLException {
return new VersionedProfile(
resultSet.getString(Profiles.VERSION),
resultSet.getString(Profiles.NAME),
resultSet.getString(Profiles.AVATAR),
resultSet.getString(Profiles.ABOUT_EMOJI),
resultSet.getString(Profiles.ABOUT),
resultSet.getString(Profiles.PAYMENT_ADDRESS),
resultSet.getBytes(Profiles.COMMITMENT));
}
}
@@ -29,7 +29,6 @@ import com.stripe.param.SetupIntentCreateParams;
import com.stripe.param.SubscriptionCancelParams;
import com.stripe.param.SubscriptionCreateParams;
import com.stripe.param.SubscriptionListParams;
import com.stripe.param.SubscriptionRetrieveParams;
import com.stripe.param.SubscriptionUpdateParams;
import com.stripe.param.SubscriptionUpdateParams.BillingCycleAnchor;
import com.stripe.param.SubscriptionUpdateParams.ProrationBehavior;
@@ -44,7 +43,6 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
@@ -54,9 +52,6 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import org.apache.commons.codec.binary.Hex;
import org.whispersystems.textsecuregcm.util.Conversions;
@@ -67,13 +62,11 @@ public class StripeManager {
private final String apiKey;
private final Executor executor;
private final byte[] idempotencyKeyGenerator;
private final String boostDescription;
public StripeManager(
@Nonnull String apiKey,
@Nonnull Executor executor,
@Nonnull byte[] idempotencyKeyGenerator,
@Nonnull String boostDescription) {
@Nonnull byte[] idempotencyKeyGenerator) {
this.apiKey = Objects.requireNonNull(apiKey);
if (Strings.isNullOrEmpty(apiKey)) {
throw new IllegalArgumentException("apiKey cannot be empty");
@@ -83,7 +76,6 @@ public class StripeManager {
if (idempotencyKeyGenerator.length == 0) {
throw new IllegalArgumentException("idempotencyKeyGenerator cannot be empty");
}
this.boostDescription = Objects.requireNonNull(boostDescription);
}
private RequestOptions commonOptions() {
@@ -151,27 +143,16 @@ public class StripeManager {
}, executor);
}
/**
* Creates a payment intent. May throw a 400 WebApplicationException if the amount is too small.
*/
public CompletableFuture<PaymentIntent> createPaymentIntent(String currency, long amount) {
return CompletableFuture.supplyAsync(() -> {
PaymentIntentCreateParams params = PaymentIntentCreateParams.builder()
.setAmount(amount)
.setCurrency(currency.toLowerCase(Locale.ROOT))
.setDescription(boostDescription)
.build();
try {
return PaymentIntent.create(params, commonOptions());
} catch (StripeException e) {
if ("amount_too_small".equalsIgnoreCase(e.getCode())) {
throw new WebApplicationException(Response
.status(Status.BAD_REQUEST)
.entity(Map.of("error", "amount_too_small"))
.build());
} else {
throw new CompletionException(e);
}
throw new CompletionException(e);
}
}, executor);
}
@@ -360,19 +341,6 @@ public class StripeManager {
}, executor);
}
public CompletableFuture<Invoice> getLatestInvoiceForSubscription(String subscriptionId) {
return CompletableFuture.supplyAsync(() -> {
SubscriptionRetrieveParams params = SubscriptionRetrieveParams.builder()
.addExpand("latest_invoice")
.build();
try {
return Subscription.retrieve(subscriptionId, params, commonOptions()).getLatestInvoiceObject();
} catch (StripeException e) {
throw new CompletionException(e);
}
}, executor);
}
public CompletableFuture<Collection<InvoiceLineItem>> getInvoiceLineItemsForInvoice(Invoice invoice) {
return CompletableFuture.supplyAsync(
() -> Lists.newArrayList(invoice.getLines().autoPagingIterable(null, commonOptions())), executor);
@@ -119,6 +119,6 @@ public class AttributeValues {
}
public static UUID getUUID(Map<String, AttributeValue> item, String key, UUID defaultValue) {
return AttributeValues.get(item, key).filter(av -> av.b() != null).map(AttributeValues::toUUID).orElse(defaultValue);
return AttributeValues.get(item, key).map(AttributeValues::toUUID).orElse(defaultValue);
}
}
@@ -21,17 +21,6 @@ public class DynamoDbFromConfig {
.build();
}
public static DynamoDbClient client(DynamoDbClientConfiguration config, AwsCredentialsProvider credentialsProvider) {
return DynamoDbClient.builder()
.region(Region.of(config.getRegion()))
.credentialsProvider(credentialsProvider)
.overrideConfiguration(ClientOverrideConfiguration.builder()
.apiCallTimeout(config.getClientExecutionTimeout())
.apiCallAttemptTimeout(config.getClientRequestTimeout())
.build())
.build();
}
public static DynamoDbAsyncClient asyncClient(
DynamoDbClientConfiguration config,
AwsCredentialsProvider credentialsProvider) {
@@ -1,27 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.util;
import static java.lang.annotation.ElementType.FIELD;
import static java.lang.annotation.ElementType.PARAMETER;
import static java.lang.annotation.RetentionPolicy.RUNTIME;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;
import javax.validation.Constraint;
import javax.validation.Payload;
@Target({ FIELD, PARAMETER })
@Retention(RUNTIME)
@Constraint(validatedBy = UsernameValidator.class)
public @interface Username {
String message() default "{org.whispersystems.textsecuregcm.util.Username.message}";
Class<?>[] groups() default { };
Class<? extends Payload>[] payload() default { };
}
@@ -1,26 +0,0 @@
/*
* Copyright 2013-2021 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.whispersystems.textsecuregcm.util;
import org.apache.commons.lang3.StringUtils;
import javax.validation.ConstraintValidator;
import javax.validation.ConstraintValidatorContext;
import java.util.regex.Pattern;
public class UsernameValidator implements ConstraintValidator<Username, String> {
private static final Pattern USERNAME_PATTERN =
Pattern.compile("^[a-z_][a-z0-9_]{3,25}$", Pattern.CASE_INSENSITIVE);
@Override
public boolean isValid(final String username, final ConstraintValidatorContext context) {
return StringUtils.isNotBlank(username) && USERNAME_PATTERN.matcher(getCanonicalUsername(username)).matches();
}
public static String getCanonicalUsername(final String username) {
return username != null ? username.toLowerCase() : null;
}
}

Some files were not shown because too many files have changed in this diff Show More