Github archive revert

backend/config/custom-environment-variables.json

@@ -104,7 +104,8 @@
     "callbackUrl": "CROWD_GITHUB_CALLBACK_URL",
     "privateKey": "CROWD_GITHUB_PRIVATE_KEY",
     "webhookSecret": "CROWD_GITHUB_WEBHOOK_SECRET",
-    "isCommitDataEnabled": "CROWD_GITHUB_IS_COMMIT_DATA_ENABLED"
+    "isCommitDataEnabled": "CROWD_GITHUB_IS_COMMIT_DATA_ENABLED",
+    "isSnowflakeEnabled": "CROWD_GITHUB_IS_SNOWFLAKE_ENABLED"
   },
   "githubIssueReporter": {
     "appId": "CROWD_GITHUB_ISSUE_REPORTER_APP_ID",

backend/package.json

@@ -32,7 +32,8 @@
     "script:purge-tenants-and-data": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/purge-tenants-and-data.ts",
     "script:import-lfx-memberships": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/import-lfx-memberships.ts",
     "script:fix-missing-org-displayName": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/fix-missing-org-displayName.ts",
-    "script:syncActivities": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/syncActivities.ts"
+    "script:syncActivities": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/syncActivities.ts",
+    "script:refreshGithubRepoSettings": "SERVICE=script TS_NODE_TRANSPILE_ONLY=true tsx src/bin/scripts/refresh-github-repo-settings.ts"
   },
   "dependencies": {
     "@aws-sdk/client-comprehend": "^3.159.0",

backend/src/api/integration/helpers/githubAuthenticate.ts

@@ -0,0 +1,13 @@
+import Permissions from '../../../security/permissions'
+import IntegrationService from '../../../services/integrationService'
+import PermissionChecker from '../../../services/user/permissionChecker'
+
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.tenantEdit)
+  const payload = await new IntegrationService(req).connectGithub(
+    req.params.code,
+    req.body.installId,
+    req.body.setupAction,
+  )
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/integration/helpers/githubConnectInstallation.ts

@@ -0,0 +1,9 @@
+import Permissions from '../../../security/permissions'
+import IntegrationService from '../../../services/integrationService'
+import PermissionChecker from '../../../services/user/permissionChecker'
+
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.tenantEdit)
+  const payload = await new IntegrationService(req).connectGithubInstallation(req.body.installId)
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/integration/helpers/githubGetInstallations.ts

@@ -0,0 +1,9 @@
+import Permissions from '../../../security/permissions'
+import IntegrationService from '../../../services/integrationService'
+import PermissionChecker from '../../../services/user/permissionChecker'
+
+export default async (req, res) => {
+  new PermissionChecker(req).validateHas(Permissions.values.tenantEdit)
+  const payload = await new IntegrationService(req).getGithubInstallations()
+  await req.responseHandler.success(req, res, payload)
+}

backend/src/api/integration/helpers/githubMapRepos.ts

@@ -1,14 +1,27 @@
+import { GITHUB_CONFIG } from '@/conf'
+
 import Permissions from '../../../security/permissions'
 import IntegrationService from '../../../services/integrationService'
 import PermissionChecker from '../../../services/user/permissionChecker'
 
+const isSnowflakeEnabled = GITHUB_CONFIG.isSnowflakeEnabled === 'true'
+
 export default async (req, res) => {
   new PermissionChecker(req).validateHas(Permissions.values.tenantEdit)
-  const payload = await new IntegrationService(req).mapGithubRepos(
-    req.params.id,
-    req.body.mapping,
-    true,
-    req.body?.isUpdateTransaction ?? false,
-  )
+  let payload
+  if (isSnowflakeEnabled) {
+    payload = await new IntegrationService(req).mapGithubReposSnowflake(
+      req.params.id,
+      req.body.mapping,
+      true,
+      req.body?.isUpdateTransaction ?? false,
+    )
+  } else {
+    payload = await new IntegrationService(req).mapGithubRepos(
+      req.params.id,
+      req.body.mapping,
+      true,
+    )
+  }
   await req.responseHandler.success(req, res, payload)
 }

backend/src/api/integration/index.ts

@@ -26,9 +26,22 @@ export default (app) => {
   app.get(`/integration/autocomplete`, safeWrap(require('./integrationAutocomplete').default))
   app.get(`/integration/global`, safeWrap(require('./integrationGlobal').default))
   app.get(`/integration/global/status`, safeWrap(require('./integrationGlobalStatus').default))
+
+  app.get(
+    '/integration/github-installations',
+    safeWrap(require('./helpers/githubGetInstallations').default),
+  )
+
+  app.post(
+    '/integration/github-connect-installation',
+    safeWrap(require('./helpers/githubConnectInstallation').default),
+  )
+
   app.get(`/integration`, safeWrap(require('./integrationList').default))
   app.get(`/integration/:id`, safeWrap(require('./integrationFind').default))
 
+  app.put(`/authenticate/:code`, safeWrap(require('./helpers/githubAuthenticate').default))
+
   app.put(`/integration/:id/github/repos`, safeWrap(require('./helpers/githubMapRepos').default))
   app.get(`/integration/:id/github/repos`, safeWrap(require('./helpers/githubMapReposGet').default))
   app.get(

backend/src/bin/jobs/refreshGithubRepoSettings.ts

@@ -4,12 +4,15 @@ import cronGenerator from 'cron-time-generator'
 import { timeout } from '@crowd/common'
 import { getServiceChildLogger } from '@crowd/logging'
 
+import { GITHUB_CONFIG } from '../../conf'
 import GithubReposRepository from '../../database/repositories/githubReposRepository'
 import SequelizeRepository from '../../database/repositories/sequelizeRepository'
 import GithubIntegrationService from '../../services/githubIntegrationService'
 import IntegrationService from '../../services/integrationService'
 import { CrowdJob } from '../../types/jobTypes'
 
+const IS_GITHUB_SNOWFLAKE_ENABLED = GITHUB_CONFIG.isSnowflakeEnabled === 'true'
+
 const log = getServiceChildLogger('refreshGithubRepoSettings')
 
 interface Integration {
@@ -33,8 +36,7 @@ interface Integration {
   }
 }
 
-export const refreshGithubRepoSettings = async () => {
-  log.info('Updating Github repo settings.')
+const refreshForSnowflake = async () => {
   const dbOptions = await SequelizeRepository.getDefaultIRepositoryOptions()
 
   const githubIntegrations = await dbOptions.database.sequelize.query(
@@ -124,7 +126,7 @@ export const refreshGithubRepoSettings = async () => {
         }
 
         // Map new repos
-        await integrationService.mapGithubRepos(
+        await integrationService.mapGithubReposSnowflake(
           integration.id,
           newFullMapping,
           true,
@@ -150,6 +152,56 @@ export const refreshGithubRepoSettings = async () => {
   log.info('Finished updating Github repo settings.')
 }
 
+const refreshForGitHub = async () => {
+  log.info('Updating Github repo settings.')
+  const dbOptions = await SequelizeRepository.getDefaultIRepositoryOptions()
+
+  interface Integration {
+    id: string
+    tenantId: string
+    integrationIdentifier: string
+  }
+
+  const githubIntegrations = await dbOptions.database.sequelize.query(
+    `SELECT id, "tenantId", "integrationIdentifier" FROM integrations 
+       WHERE platform = 'github' AND "deletedAt" IS NULL
+       AND "createdAt" < NOW() - INTERVAL '1 minute' AND "integrationIdentifier" IS NOT NULL`,
+  )
+
+  for (const integration of githubIntegrations[0] as Integration[]) {
+    log.info(`Updating repo settings for Github integration: ${integration.id}`)
+
+    try {
+      const options = await SequelizeRepository.getDefaultIRepositoryOptions()
+      options.currentTenant = { id: integration.tenantId }
+
+      const integrationService = new IntegrationService(options)
+      // newly discovered repos will be mapped to default segment of the integration
+      await integrationService.updateGithubIntegrationSettings(integration.integrationIdentifier)
+
+      log.info(`Successfully updated repo settings for Github integration: ${integration.id}`)
+    } catch (err) {
+      log.error(
+        `Error updating repo settings for Github integration ${integration.id}: ${err.message}`,
+      )
+    } finally {
+      await timeout(1000)
+    }
+  }
+
+  log.info('Finished updating Github repo settings.')
+}
+
+export const refreshGithubRepoSettings = async () => {
+  log.info('Updating Github repo settings.')
+
+  if (IS_GITHUB_SNOWFLAKE_ENABLED) {
+    await refreshForSnowflake()
+  } else {
+    await refreshForGitHub()
+  }
+}
+
 const job: CrowdJob = {
   name: 'Refresh Github repo settings',
   // every day

backend/src/conf/configTypes.ts

@@ -120,6 +120,7 @@ export interface GithubConfiguration {
   privateKey: string
   webhookSecret: string
   isCommitDataEnabled: string
+  isSnowflakeEnabled: string
   globalLimit?: number
   callbackUrl: string
 }

backend/src/database/repositories/githubReposRepository.ts

@@ -38,7 +38,24 @@ export default class GithubReposRepository {
     )
   }
 
-  static async updateMapping(
+  static async updateMapping(integrationId, mapping, options: IRepositoryOptions) {
+    const tenantId = options.currentTenant.id
+
+    await GithubReposRepository.bulkInsert(
+      'githubRepos',
+      ['tenantId', 'integrationId', 'segmentId', 'url'],
+      (idx) => `(:tenantId_${idx}, :integrationId_${idx}, :segmentId_${idx}, :url_${idx})`,
+      Object.entries(mapping).map(([url, segmentId], idx) => ({
+        [`tenantId_${idx}`]: tenantId,
+        [`integrationId_${idx}`]: integrationId,
+        [`segmentId_${idx}`]: segmentId,
+        [`url_${idx}`]: url,
+      })),
+      options,
+    )
+  }
+
+  static async updateMappingSnowflake(
     integrationId,
     newMapping: Record<string, string>,
     oldMapping: {

backend/src/serverless/integrations/types/regularTypes.ts

@@ -3,7 +3,13 @@ import { PlatformType } from '@crowd/types'
 export type Repo = {
   url: string
   name: string
-  updatedAt: string
+  updatedAt?: string
+  createdAt?: string
+  owner?: string
+  available?: boolean
+  fork?: boolean
+  private?: boolean
+  cloneUrl?: string
 }
 
 export type Repos = Array<Repo>

backend/src/serverless/integrations/usecases/github/rest/getInstalledRepositories.ts

@@ -0,0 +1,64 @@
+import axios, { AxiosRequestConfig } from 'axios'
+
+import { getServiceChildLogger } from '@crowd/logging'
+
+import { Repos } from '../../../types/regularTypes'
+
+const log = getServiceChildLogger('getInstalledRepositories')
+
+const getRepositoriesFromGH = async (page: number, installToken: string): Promise<any> => {
+  const REPOS_PER_PAGE = 100
+
+  const requestConfig = {
+    method: 'get',
+    url: `https://api.github.com/installation/repositories?page=${page}&per_page=${REPOS_PER_PAGE}`,
+    headers: {
+      Authorization: `Bearer ${installToken}`,
+    },
+  } as AxiosRequestConfig
+
+  const response = await axios(requestConfig)
+  return response.data
+}
+
+const parseRepos = (repositories: any): Repos => {
+  const repos: Repos = []
+
+  for (const repo of repositories) {
+    repos.push({
+      url: repo.html_url,
+      owner: repo.owner.login,
+      createdAt: repo.created_at,
+      name: repo.name,
+      fork: repo.fork,
+      private: repo.private,
+      cloneUrl: repo.clone_url,
+    })
+  }
+
+  return repos
+}
+
+export const getInstalledRepositories = async (installToken: string): Promise<Repos> => {
+  try {
+    let page = 1
+    let hasMorePages = true
+
+    const repos: Repos = []
+
+    while (hasMorePages) {
+      const data = await getRepositoriesFromGH(page, installToken)
+
+      if (data.repositories) {
+        repos.push(...parseRepos(data.repositories))
+      }
+
+      hasMorePages = data.total_count && data.total_count > 0 && data.total_count > repos.length
+      page += 1
+    }
+    return repos.filter((repo) => !repo.private && !repo.fork)
+  } catch (err: any) {
+    log.error(err, 'Error fetching installed repositories!')
+    throw err
+  }
+}