|
| 1 | +package conformance_test |
| 2 | + |
| 3 | +import ( |
| 4 | + "bufio" |
| 5 | + "context" |
| 6 | + "encoding/json" |
| 7 | + "fmt" |
| 8 | + "io" |
| 9 | + "os" |
| 10 | + "strings" |
| 11 | + "testing" |
| 12 | + |
| 13 | + "github.com/docker/docker/api/types/container" |
| 14 | + "github.com/docker/docker/api/types/network" |
| 15 | + "github.com/docker/docker/client" |
| 16 | + "github.com/docker/docker/pkg/stdcopy" |
| 17 | + "github.com/google/go-cmp/cmp" |
| 18 | + "github.com/stretchr/testify/require" |
| 19 | +) |
| 20 | + |
| 21 | +type maintainer string |
| 22 | + |
| 23 | +const ( |
| 24 | + anthropic maintainer = "anthropic" |
| 25 | + github maintainer = "github" |
| 26 | +) |
| 27 | + |
| 28 | +type testLogWriter struct { |
| 29 | + t *testing.T |
| 30 | +} |
| 31 | + |
| 32 | +func (w testLogWriter) Write(p []byte) (n int, err error) { |
| 33 | + w.t.Log(string(p)) |
| 34 | + return len(p), nil |
| 35 | +} |
| 36 | + |
| 37 | +func start(t *testing.T, m maintainer) server { |
| 38 | + var image string |
| 39 | + if m == github { |
| 40 | + image = "github/github-mcp-server" |
| 41 | + } else { |
| 42 | + image = "mcp/github" |
| 43 | + } |
| 44 | + |
| 45 | + ctx := context.Background() |
| 46 | + dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) |
| 47 | + require.NoError(t, err) |
| 48 | + |
| 49 | + containerCfg := &container.Config{ |
| 50 | + OpenStdin: true, |
| 51 | + AttachStdin: true, |
| 52 | + AttachStdout: true, |
| 53 | + AttachStderr: true, |
| 54 | + Env: []string{ |
| 55 | + fmt.Sprintf("GITHUB_PERSONAL_ACCESS_TOKEN=%s", os.Getenv("GITHUB_PERSONAL_ACCESS_TOKEN")), |
| 56 | + }, |
| 57 | + Image: image, |
| 58 | + } |
| 59 | + |
| 60 | + resp, err := dockerClient.ContainerCreate( |
| 61 | + ctx, |
| 62 | + containerCfg, |
| 63 | + &container.HostConfig{}, |
| 64 | + &network.NetworkingConfig{}, |
| 65 | + nil, |
| 66 | + "") |
| 67 | + require.NoError(t, err) |
| 68 | + |
| 69 | + t.Cleanup(func() { |
| 70 | + require.NoError(t, dockerClient.ContainerRemove(ctx, resp.ID, container.RemoveOptions{Force: true})) |
| 71 | + }) |
| 72 | + |
| 73 | + hijackedResponse, err := dockerClient.ContainerAttach(ctx, resp.ID, container.AttachOptions{ |
| 74 | + Stream: true, |
| 75 | + Stdin: true, |
| 76 | + Stdout: true, |
| 77 | + Stderr: true, |
| 78 | + }) |
| 79 | + require.NoError(t, err) |
| 80 | + t.Cleanup(func() { hijackedResponse.Close() }) |
| 81 | + |
| 82 | + require.NoError(t, dockerClient.ContainerStart(ctx, resp.ID, container.StartOptions{})) |
| 83 | + |
| 84 | + serverStart := make(chan serverStartResult) |
| 85 | + go func() { |
| 86 | + prOut, pwOut := io.Pipe() |
| 87 | + prErr, pwErr := io.Pipe() |
| 88 | + |
| 89 | + go func() { |
| 90 | + // Ignore error, we should be done? |
| 91 | + // TODO: maybe check for use of closed network connection specifically |
| 92 | + _, _ = stdcopy.StdCopy(pwOut, pwErr, hijackedResponse.Reader) |
| 93 | + pwOut.Close() |
| 94 | + pwErr.Close() |
| 95 | + }() |
| 96 | + |
| 97 | + bufferedStderr := bufio.NewReader(prErr) |
| 98 | + line, err := bufferedStderr.ReadString('\n') |
| 99 | + if err != nil { |
| 100 | + serverStart <- serverStartResult{err: err} |
| 101 | + } |
| 102 | + |
| 103 | + if strings.TrimSpace(line) != "GitHub MCP Server running on stdio" { |
| 104 | + serverStart <- serverStartResult{ |
| 105 | + err: fmt.Errorf("unexpected server output: %s", line), |
| 106 | + } |
| 107 | + return |
| 108 | + } |
| 109 | + |
| 110 | + serverStart <- serverStartResult{ |
| 111 | + server: server{ |
| 112 | + m: m, |
| 113 | + log: testLogWriter{t}, |
| 114 | + stdin: hijackedResponse.Conn, |
| 115 | + stdout: bufio.NewReader(prOut), |
| 116 | + }, |
| 117 | + } |
| 118 | + }() |
| 119 | + |
| 120 | + t.Logf("waiting for %s server to start...", m) |
| 121 | + serveResult := <-serverStart |
| 122 | + require.NoError(t, serveResult.err, "expected the server to start successfully") |
| 123 | + |
| 124 | + return serveResult.server |
| 125 | +} |
| 126 | + |
| 127 | +func TestCapabilities(t *testing.T) { |
| 128 | + githubServer := start(t, github) |
| 129 | + anthropicServer := start(t, anthropic) |
| 130 | + |
| 131 | + req := newInitializeRequest( |
| 132 | + initializeRequestParams{ |
| 133 | + ProtocolVersion: "2024-11-05", |
| 134 | + Capabilities: clientCapabilities{}, |
| 135 | + ClientInfo: clientInfo{ |
| 136 | + Name: "ConformanceTest", |
| 137 | + Version: "0.0.1", |
| 138 | + }, |
| 139 | + }, |
| 140 | + ) |
| 141 | + |
| 142 | + require.NoError(t, githubServer.send(req)) |
| 143 | + |
| 144 | + var ghInitializeResponse initializeResponse |
| 145 | + require.NoError(t, githubServer.receive(&ghInitializeResponse)) |
| 146 | + |
| 147 | + require.NoError(t, anthropicServer.send(req)) |
| 148 | + |
| 149 | + var anthropicInitializeResponse initializeResponse |
| 150 | + require.NoError(t, anthropicServer.receive(&anthropicInitializeResponse)) |
| 151 | + |
| 152 | + if diff := cmp.Diff(ghInitializeResponse.Result.Capabilities, anthropicInitializeResponse.Result.Capabilities); diff != "" { |
| 153 | + t.Errorf("unexpected capability differential: %s", diff) |
| 154 | + } |
| 155 | +} |
| 156 | + |
| 157 | +type serverStartResult struct { |
| 158 | + server server |
| 159 | + err error |
| 160 | +} |
| 161 | + |
| 162 | +type server struct { |
| 163 | + m maintainer |
| 164 | + log io.Writer |
| 165 | + |
| 166 | + stdin io.Writer |
| 167 | + stdout *bufio.Reader |
| 168 | +} |
| 169 | + |
| 170 | +func (s server) send(req request) error { |
| 171 | + b, err := req.marshal() |
| 172 | + if err != nil { |
| 173 | + return err |
| 174 | + } |
| 175 | + |
| 176 | + fmt.Fprintf(s.log, "sending %s: %s\n", s.m, string(b)) |
| 177 | + |
| 178 | + n, err := s.stdin.Write(append(b, '\n')) |
| 179 | + if err != nil { |
| 180 | + return err |
| 181 | + } |
| 182 | + |
| 183 | + if n != len(b)+1 { |
| 184 | + return fmt.Errorf("wrote %d bytes, expected %d", n, len(b)+1) |
| 185 | + } |
| 186 | + |
| 187 | + return nil |
| 188 | +} |
| 189 | + |
| 190 | +func (s server) receive(res response) error { |
| 191 | + line, err := s.stdout.ReadBytes('\n') |
| 192 | + if err != nil { |
| 193 | + if err == io.EOF { |
| 194 | + return fmt.Errorf("EOF after reading %s", string(line)) |
| 195 | + } |
| 196 | + return err |
| 197 | + } |
| 198 | + |
| 199 | + fmt.Fprintf(s.log, "received from %s: %s\n", s.m, string(line)) |
| 200 | + |
| 201 | + return res.unmarshal(line) |
| 202 | +} |
| 203 | + |
| 204 | +type jsonRPRCRequest[params any] struct { |
| 205 | + JSONRPC string `json:"jsonrpc"` |
| 206 | + ID int `json:"id"` |
| 207 | + Method string `json:"method"` |
| 208 | + Params params `json:"params"` |
| 209 | +} |
| 210 | + |
| 211 | +type jsonRPRCResponse[result any] struct { |
| 212 | + JSONRPC string `json:"jsonrpc"` |
| 213 | + ID int `json:"id"` |
| 214 | + Method string `json:"method"` |
| 215 | + Result result `json:"result"` |
| 216 | +} |
| 217 | + |
| 218 | +type request interface { |
| 219 | + marshal() ([]byte, error) |
| 220 | +} |
| 221 | + |
| 222 | +type response interface { |
| 223 | + unmarshal([]byte) error |
| 224 | +} |
| 225 | + |
| 226 | +func newInitializeRequest(params initializeRequestParams) initializeRequest { |
| 227 | + return initializeRequest{ |
| 228 | + jsonRPRCRequest: jsonRPRCRequest[initializeRequestParams]{ |
| 229 | + JSONRPC: "2.0", |
| 230 | + ID: 1, |
| 231 | + Method: "initialize", |
| 232 | + Params: params, |
| 233 | + }, |
| 234 | + } |
| 235 | +} |
| 236 | + |
| 237 | +type initializeRequest struct { |
| 238 | + jsonRPRCRequest[initializeRequestParams] |
| 239 | +} |
| 240 | + |
| 241 | +func (r initializeRequest) marshal() ([]byte, error) { |
| 242 | + return json.Marshal(r) |
| 243 | +} |
| 244 | + |
| 245 | +type initializeRequestParams struct { |
| 246 | + ProtocolVersion string `json:"protocolVersion"` |
| 247 | + Capabilities clientCapabilities `json:"capabilities"` |
| 248 | + ClientInfo clientInfo `json:"clientInfo"` |
| 249 | +} |
| 250 | + |
| 251 | +type clientCapabilities struct{} // don't actually care about any of these right now |
| 252 | + |
| 253 | +type clientInfo struct { |
| 254 | + Name string `json:"name"` |
| 255 | + Version string `json:"version"` |
| 256 | +} |
| 257 | + |
| 258 | +type initializeResponse struct { |
| 259 | + jsonRPRCResponse[initializeResult] |
| 260 | +} |
| 261 | + |
| 262 | +func (r *initializeResponse) unmarshal(b []byte) error { |
| 263 | + return json.Unmarshal(b, r) |
| 264 | +} |
| 265 | + |
| 266 | +type initializeResult struct { |
| 267 | + ProtocolVersion string `json:"protocolVersion"` |
| 268 | + Capabilities serverCapabilities `json:"capabilities"` |
| 269 | + ServerInfo serverInfo `json:"serverInfo"` |
| 270 | +} |
| 271 | + |
| 272 | +type serverCapabilities struct { |
| 273 | + Logging struct{} `json:"logging"` |
| 274 | + Prompts struct { |
| 275 | + ListChanged bool `json:"listChanged"` |
| 276 | + } `json:"prompts"` |
| 277 | + Resources struct { |
| 278 | + Subscribe bool `json:"subscribe"` |
| 279 | + ListChanged bool `json:"listChanged"` |
| 280 | + } `json:"resources"` |
| 281 | + Tools struct { |
| 282 | + ListChanged bool `json:"listChanged"` |
| 283 | + } `json:"tools"` |
| 284 | +} |
| 285 | + |
| 286 | +type serverInfo struct { |
| 287 | + Name string `json:"name"` |
| 288 | + Version string `json:"version"` |
| 289 | +} |
0 commit comments