Merge pull request #3169 from cdr/jsjoeio/add-terminal-e2e-test

feat(testing): add e2e tests for code-server and terminal

src/node/constants.ts

@@ -1,5 +1,6 @@
 import { logger } from "@coder/logger"
 import { JSONSchemaForNPMPackageJsonFiles } from "@schemastore/package"
+import * as os from "os"
 import * as path from "path"
 
 export function getPackageJson(relativePath: string): JSONSchemaForNPMPackageJsonFiles {
@@ -18,3 +19,4 @@ const pkg = getPackageJson("../../package.json")
 export const version = pkg.version || "development"
 export const commit = pkg.commit || "development"
 export const rootPath = path.resolve(__dirname, "../..")
+export const tmpdir = path.join(os.tmpdir(), "code-server")

src/node/socket.ts

@@ -4,7 +4,8 @@ import * as path from "path"
 import * as tls from "tls"
 import { Emitter } from "../common/emitter"
 import { generateUuid } from "../common/util"
-import { canConnect, tmpdir } from "./util"
+import { tmpdir } from "./constants"
+import { canConnect } from "./util"
 
 /**
  * Provides a way to proxy a TLS socket. Can be used when you need to pass a

src/node/util.ts

@@ -8,8 +8,6 @@ import * as path from "path"
 import * as util from "util"
 import xdgBasedir from "xdg-basedir"
 
-export const tmpdir = path.join(os.tmpdir(), "code-server")
-
 interface Paths {
   data: string
   config: string

test/e2e/browser.test.ts

@@ -1,15 +1,23 @@
 import { test, expect } from "@playwright/test"
-import { CODE_SERVER_ADDRESS } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
 
 // This is a "gut-check" test to make sure playwright is working as expected
-test("browser should display correct userAgent", async ({ page, browserName }) => {
+test.describe("browser", () => {
-  const displayNames = {
+  let codeServer: CodeServer
-    chromium: "Chrome",
-    firefox: "Firefox",
-    webkit: "Safari",
-  }
-  await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
-  const userAgent = await page.evaluate("navigator.userAgent")
 
-  expect(userAgent).toContain(displayNames[browserName])
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.navigate()
+  })
+
+  test("browser should display correct userAgent", async ({ page, browserName }) => {
+    const displayNames = {
+      chromium: "Chrome",
+      firefox: "Firefox",
+      webkit: "Safari",
+    }
+    const userAgent = await page.evaluate("navigator.userAgent")
+
+    expect(userAgent).toContain(displayNames[browserName])
+  })
 })

test/e2e/codeServer.test.ts

@@ -0,0 +1,45 @@
+import { test, expect } from "@playwright/test"
+import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
+
+test.describe("CodeServer", () => {
+  // Create a new context with the saved storage state
+  // so we don't have to logged in
+  const options: any = {}
+  let codeServer: CodeServer
+
+  // TODO@jsjoeio
+  // Fix this once https://github.com/microsoft/playwright-test/issues/240
+  // is fixed
+  if (STORAGE) {
+    const storageState = JSON.parse(STORAGE) || {}
+    options.contextOptions = {
+      storageState,
+    }
+  }
+
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.setup()
+  })
+
+  test(`should navigate to ${CODE_SERVER_ADDRESS}`, options, async ({ page }) => {
+    // We navigate codeServer before each test
+    // and we start the test with a storage state
+    // which means we should be logged in
+    // so it should be on the address
+    const url = page.url()
+    // We use match because there may be a / at the end
+    // so we don't want it to fail if we expect http://localhost:8080 to match http://localhost:8080/
+    expect(url).toMatch(CODE_SERVER_ADDRESS)
+  })
+
+  test("should always see the code-server editor", options, async ({ page }) => {
+    expect(await codeServer.isEditorVisible()).toBe(true)
+  })
+
+  test("should show the Integrated Terminal", options, async ({ page }) => {
+    await codeServer.focusTerminal()
+    expect(await page.isVisible("#terminal")).toBe(true)
+  })
+})

test/e2e/globalSetup.test.ts

@@ -1,5 +1,6 @@
 import { test, expect } from "@playwright/test"
-import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
+import { STORAGE } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
 
 // This test is to make sure the globalSetup works as expected
 // meaning globalSetup ran and stored the storageState in STORAGE
@@ -7,6 +8,7 @@ test.describe("globalSetup", () => {
   // Create a new context with the saved storage state
   // so we don't have to logged in
   const options: any = {}
+  let codeServer: CodeServer
 
   // TODO@jsjoeio
   // Fix this once https://github.com/microsoft/playwright-test/issues/240
@@ -17,9 +19,12 @@ test.describe("globalSetup", () => {
       storageState,
     }
   }
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.setup()
+  })
   test("should keep us logged in using the storageState", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Make sure the editor actually loaded
-    expect(await page.isVisible("div.monaco-workbench"))
+    expect(await codeServer.isEditorVisible()).toBe(true)
   })
 })

test/e2e/login.test.ts

@@ -1,5 +1,6 @@
 import { test, expect } from "@playwright/test"
-import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
+import { PASSWORD } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
 
 test.describe("login", () => {
   // Reset the browser so no cookies are persisted
@@ -9,26 +10,32 @@ test.describe("login", () => {
       storageState: {},
     },
   }
+  let codeServer: CodeServer
+
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.navigate()
+  })
 
   test("should see the login page", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // It should send us to the login page
     expect(await page.title()).toBe("code-server login")
   })
 
   test("should be able to login", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login
     await page.click(".submit")
     await page.waitForLoadState("networkidle")
+    // We do this because occassionally code-server doesn't load on Firefox
+    // but loads if you reload once or twice
+    await codeServer.reloadUntilEditorIsVisible()
     // Make sure the editor actually loaded
-    expect(await page.isVisible("div.monaco-workbench"))
+    expect(await codeServer.isEditorVisible()).toBe(true)
   })
 
   test("should see an error message for missing password", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Skip entering password
     // Click the submit button and login
     await page.click(".submit")
@@ -37,7 +44,6 @@ test.describe("login", () => {
   })
 
   test("should see an error message for incorrect password", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Type in password
     await page.fill(".password", "password123")
     // Click the submit button and login
@@ -47,7 +53,6 @@ test.describe("login", () => {
   })
 
   test("should hit the rate limiter for too many unsuccessful logins", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Type in password
     await page.fill(".password", "password123")
     // Click the submit button and login

test/e2e/logout.test.ts

@@ -1,5 +1,6 @@
 import { test, expect } from "@playwright/test"
 import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
 
 test.describe("logout", () => {
   // Reset the browser so no cookies are persisted
@@ -9,22 +10,31 @@ test.describe("logout", () => {
       storageState: {},
     },
   }
+  let codeServer: CodeServer
+
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.navigate()
+  })
+
   test("should be able login and logout", options, async ({ page }) => {
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login
     await page.click(".submit")
     await page.waitForLoadState("networkidle")
+    // We do this because occassionally code-server doesn't load on Firefox
+    // but loads if you reload once or twice
+    await codeServer.reloadUntilEditorIsVisible()
     // Make sure the editor actually loaded
-    expect(await page.isVisible("div.monaco-workbench"))
+    expect(await codeServer.isEditorVisible()).toBe(true)
 
     // Click the Application menu
     await page.click("[aria-label='Application Menu']")
 
     // See the Log out button
     const logoutButton = "a.action-menu-item span[aria-label='Log out']"
-    expect(await page.isVisible(logoutButton))
+    expect(await page.isVisible(logoutButton)).toBe(true)
 
     await page.hover(logoutButton)
     // TODO(@jsjoeio)

test/e2e/models/CodeServer.ts

@@ -0,0 +1,104 @@
+import { Page } from "playwright"
+import { CODE_SERVER_ADDRESS } from "../../utils/constants"
+// This is a Page Object Model
+// We use these to simplify e2e test authoring
+// See Playwright docs: https://playwright.dev/docs/pom/
+export class CodeServer {
+  page: Page
+
+  constructor(page: Page) {
+    this.page = page
+  }
+
+  /**
+   * Navigates to CODE_SERVER_ADDRESS
+   */
+  async navigate() {
+    await this.page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+  }
+
+  /**
+   * Checks if the editor is visible
+   * and reloads until it is
+   */
+  async reloadUntilEditorIsVisible() {
+    const editorIsVisible = await this.isEditorVisible()
+    let reloadCount = 0
+
+    // Occassionally code-server timeouts in Firefox
+    // we're not sure why
+    // but usually a reload or two fixes it
+    // TODO@jsjoeio @oxy look into Firefox reconnection/timeout issues
+    while (!editorIsVisible) {
+      reloadCount += 1
+      if (await this.isEditorVisible()) {
+        console.log(`    Editor became visible after ${reloadCount} reloads`)
+        break
+      }
+      // When a reload happens, we want to wait for all resources to be
+      // loaded completely. Hence why we use that instead of DOMContentLoaded
+      // Read more: https://thisthat.dev/dom-content-loaded-vs-load/
+      await this.page.reload({ waitUntil: "load" })
+    }
+  }
+
+  /**
+   * Checks if the editor is visible
+   */
+  async isEditorVisible() {
+    // Make sure the editor actually loaded
+    // If it's not visible after 5 seconds, something is wrong
+    await this.page.waitForLoadState("networkidle")
+    return await this.page.isVisible("div.monaco-workbench", { timeout: 5000 })
+  }
+
+  /**
+   * Focuses Integrated Terminal
+   * by going to the Application Menu
+   * and clicking View > Terminal
+   */
+  async focusTerminal() {
+    // If the terminal is already visible
+    // then we can focus it by hitting the keyboard shortcut
+    const isTerminalVisible = await this.page.isVisible("#terminal")
+    if (isTerminalVisible) {
+      await this.page.keyboard.press(`Control+Backquote`)
+      // Wait for terminal to receive focus
+      await this.page.waitForSelector("div.terminal.xterm.focus")
+      // Sometimes the terminal reloads
+      // which is why we wait for it twice
+      await this.page.waitForSelector("div.terminal.xterm.focus")
+      return
+    }
+    // Open using the manu
+    // Click [aria-label="Application Menu"] div[role="none"]
+    await this.page.click('[aria-label="Application Menu"] div[role="none"]')
+
+    // Click text=View
+    await this.page.hover("text=View")
+    await this.page.click("text=View")
+
+    // Click text=Terminal
+    await this.page.hover("text=Terminal")
+    await this.page.click("text=Terminal")
+
+    // Wait for terminal to receive focus
+    // Sometimes the terminal reloads once or twice
+    // which is why we wait for it to have the focus class
+    await this.page.waitForSelector("div.terminal.xterm.focus")
+    // Sometimes the terminal reloads
+    // which is why we wait for it twice
+    await this.page.waitForSelector("div.terminal.xterm.focus")
+  }
+
+  /**
+   * Navigates to CODE_SERVER_ADDRESS
+   * and reloads until the editor is visible
+   *
+   * Helpful for running before tests
+   */
+  async setup() {
+    await this.navigate()
+    await this.reloadUntilEditorIsVisible()
+  }
+}

test/e2e/openHelpAbout.test.ts

@@ -1,10 +1,12 @@
 import { test, expect } from "@playwright/test"
-import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
+import { STORAGE } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
 
 test.describe("Open Help > About", () => {
   // Create a new context with the saved storage state
   // so we don't have to logged in
   const options: any = {}
+  let codeServer: CodeServer
   // TODO@jsjoeio
   // Fix this once https://github.com/microsoft/playwright-test/issues/240
   // is fixed
@@ -15,32 +17,30 @@ test.describe("Open Help > About", () => {
     }
   }
 
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.setup()
+  })
+
   test(
     "should see a 'Help' then 'About' button in the Application Menu that opens a dialog",
     options,
     async ({ page }) => {
-      await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+      // Open using the manu
-      // Make sure the editor actually loaded
+      // Click [aria-label="Application Menu"] div[role="none"]
-      expect(await page.isVisible("div.monaco-workbench"))
+      await page.click('[aria-label="Application Menu"] div[role="none"]')
 
-      // Click the Application menu
+      // Click the Help button
-      await page.click("[aria-label='Application Menu']")
+      await page.hover("text=Help")
-      // See the Help button
+      await page.click("text=Help")
-      const helpButton = "a.action-menu-item span[aria-label='Help']"
-      expect(await page.isVisible(helpButton))
 
-      // Hover the helpButton
+      // Click the About button
-      await page.hover(helpButton)
+      await page.hover("text=About")
+      await page.click("text=About")
 
-      // see the About button and click it
+      // Click div[role="dialog"] >> text=code-server
-      const aboutButton = "a.action-menu-item span[aria-label='About']"
+      const element = await page.waitForSelector('div[role="dialog"] >> text=code-server')
-      expect(await page.isVisible(aboutButton))
+      expect(element).not.toBeNull()
-      // NOTE: it won't work unless you hover it first
-      await page.hover(aboutButton)
-      await page.click(aboutButton)
-
-      const codeServerText = "text=code-server"
-      expect(await page.isVisible(codeServerText))
     },
   )
 })

test/e2e/terminal.test.ts

@@ -0,0 +1,59 @@
+import { test, expect } from "@playwright/test"
+import * as cp from "child_process"
+import * as fs from "fs"
+// import { tmpdir } from "os"
+import * as path from "path"
+import util from "util"
+import { STORAGE, tmpdir } from "../utils/constants"
+import { CodeServer } from "./models/CodeServer"
+
+test.describe("Integrated Terminal", () => {
+  // Create a new context with the saved storage state
+  // so we don't have to logged in
+  const options: any = {}
+  const testFileName = "pipe"
+  const testString = "new string test from e2e test"
+  let codeServer: CodeServer
+  let tmpFolderPath = ""
+  let tmpFile = ""
+
+  // TODO@jsjoeio
+  // Fix this once https://github.com/microsoft/playwright-test/issues/240
+  // is fixed
+  if (STORAGE) {
+    const storageState = JSON.parse(STORAGE) || {}
+    options.contextOptions = {
+      storageState,
+    }
+  }
+  test.beforeAll(async () => {
+    tmpFolderPath = await tmpdir("integrated-terminal")
+    tmpFile = path.join(tmpFolderPath, testFileName)
+  })
+
+  test.beforeEach(async ({ page }) => {
+    codeServer = new CodeServer(page)
+    await codeServer.setup()
+  })
+
+  test.afterAll(async () => {
+    // Ensure directory was removed
+    await fs.promises.rmdir(tmpFolderPath, { recursive: true })
+  })
+
+  test("should echo a string to a file", options, async ({ page }) => {
+    const command = `mkfifo '${tmpFile}' && cat '${tmpFile}'`
+    const exec = util.promisify(cp.exec)
+    const output = exec(command, { encoding: "utf8" })
+
+    // Open terminal and type in value
+    await codeServer.focusTerminal()
+
+    await page.waitForLoadState("load")
+    await page.keyboard.type(`echo '${testString}' > '${tmpFile}'`)
+    await page.keyboard.press("Enter")
+
+    const { stdout } = await output
+    expect(stdout).toMatch(testString)
+  })
+})

test/unit/cli.test.ts

@@ -4,7 +4,8 @@ import * as net from "net"
 import * as os from "os"
 import * as path from "path"
 import { Args, parse, setDefaults, shouldOpenInExistingInstance } from "../../src/node/cli"
-import { paths, tmpdir } from "../../src/node/util"
+import { tmpdir } from "../../src/node/constants"
+import { paths } from "../../src/node/util"
 
 type Mutable<T> = {
   -readonly [P in keyof T]: T[P]

test/unit/constants.test.ts

@@ -1,4 +1,6 @@
+import * as fs from "fs"
 import { commit, getPackageJson, version } from "../../src/node/constants"
+import { tmpdir } from "../../test/utils/constants"
 import { loggerModule } from "../utils/helpers"
 
 // jest.mock is hoisted above the imports so we must use `require` here.
@@ -51,3 +53,16 @@ describe("constants", () => {
     })
   })
 })
+
+describe("test constants", () => {
+  describe("tmpdir", () => {
+    it("should return a temp directory", async () => {
+      const testName = "temp-dir"
+      const pathToTempDir = await tmpdir(testName)
+
+      expect(pathToTempDir).toContain(testName)
+
+      await fs.promises.rmdir(pathToTempDir)
+    })
+  })
+})

test/unit/socket.test.ts

@@ -4,8 +4,9 @@ import * as net from "net"
 import * as path from "path"
 import * as tls from "tls"
 import { Emitter } from "../../src/common/emitter"
+import { tmpdir } from "../../src/node/constants"
 import { SocketProxyProvider } from "../../src/node/socket"
-import { generateCertificate, tmpdir } from "../../src/node/util"
+import { generateCertificate } from "../../src/node/util"
 
 describe("SocketProxyProvider", () => {
   const provider = new SocketProxyProvider()

test/unit/update.test.ts

@@ -1,9 +1,9 @@
 import { promises as fs } from "fs"
 import * as http from "http"
 import * as path from "path"
+import { tmpdir } from "../../src/node/constants"
 import { SettingsProvider, UpdateSettings } from "../../src/node/settings"
 import { LatestResponse, UpdateProvider } from "../../src/node/update"
-import { tmpdir } from "../../src/node/util"
 
 describe.skip("update", () => {
   let version = "1.0.0"

test/utils/constants.ts

@@ -1,3 +1,14 @@
+import * as fs from "fs"
+import * as os from "os"
+import * as path from "path"
+
 export const CODE_SERVER_ADDRESS = process.env.CODE_SERVER_ADDRESS || "http://localhost:8080"
 export const PASSWORD = process.env.PASSWORD || "e45432jklfdsab"
 export const STORAGE = process.env.STORAGE || ""
+
+export async function tmpdir(testName: string): Promise<string> {
+  const dir = path.join(os.tmpdir(), "code-server")
+  await fs.promises.mkdir(dir, { recursive: true })
+
+  return await fs.promises.mkdtemp(path.join(dir, `test-${testName}-`), { encoding: "utf8" })
+}