|
| 1 | +package conformance_test |
| 2 | + |
| 3 | +import ( |
| 4 | + "bufio" |
| 5 | + "context" |
| 6 | + "encoding/json" |
| 7 | + "fmt" |
| 8 | + "io" |
| 9 | + "os" |
| 10 | + "reflect" |
| 11 | + "strings" |
| 12 | + "testing" |
| 13 | + |
| 14 | + "github.com/docker/docker/api/types/container" |
| 15 | + "github.com/docker/docker/api/types/network" |
| 16 | + "github.com/docker/docker/client" |
| 17 | + "github.com/docker/docker/pkg/stdcopy" |
| 18 | + "github.com/stretchr/testify/require" |
| 19 | +) |
| 20 | + |
| 21 | +type maintainer string |
| 22 | + |
| 23 | +const ( |
| 24 | + anthropic maintainer = "anthropic" |
| 25 | + github maintainer = "github" |
| 26 | +) |
| 27 | + |
| 28 | +type testLogWriter struct { |
| 29 | + t *testing.T |
| 30 | +} |
| 31 | + |
| 32 | +func (w testLogWriter) Write(p []byte) (n int, err error) { |
| 33 | + w.t.Log(string(p)) |
| 34 | + return len(p), nil |
| 35 | +} |
| 36 | + |
| 37 | +func start(t *testing.T, m maintainer) server { |
| 38 | + var image string |
| 39 | + if m == github { |
| 40 | + image = "github/github-mcp-server" |
| 41 | + } else { |
| 42 | + image = "mcp/github" |
| 43 | + } |
| 44 | + |
| 45 | + ctx := context.Background() |
| 46 | + dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) |
| 47 | + require.NoError(t, err) |
| 48 | + |
| 49 | + containerCfg := &container.Config{ |
| 50 | + OpenStdin: true, |
| 51 | + AttachStdin: true, |
| 52 | + AttachStdout: true, |
| 53 | + AttachStderr: true, |
| 54 | + Env: []string{ |
| 55 | + fmt.Sprintf("GITHUB_PERSONAL_ACCESS_TOKEN=%s", os.Getenv("GITHUB_PERSONAL_ACCESS_TOKEN")), |
| 56 | + }, |
| 57 | + Image: image, |
| 58 | + } |
| 59 | + |
| 60 | + resp, err := dockerClient.ContainerCreate( |
| 61 | + ctx, |
| 62 | + containerCfg, |
| 63 | + &container.HostConfig{}, |
| 64 | + &network.NetworkingConfig{}, |
| 65 | + nil, |
| 66 | + "") |
| 67 | + require.NoError(t, err) |
| 68 | + |
| 69 | + t.Cleanup(func() { |
| 70 | + require.NoError(t, dockerClient.ContainerRemove(ctx, resp.ID, container.RemoveOptions{Force: true})) |
| 71 | + }) |
| 72 | + |
| 73 | + hijackedResponse, err := dockerClient.ContainerAttach(ctx, resp.ID, container.AttachOptions{ |
| 74 | + Stream: true, |
| 75 | + Stdin: true, |
| 76 | + Stdout: true, |
| 77 | + Stderr: true, |
| 78 | + }) |
| 79 | + require.NoError(t, err) |
| 80 | + t.Cleanup(func() { hijackedResponse.Close() }) |
| 81 | + |
| 82 | + require.NoError(t, dockerClient.ContainerStart(ctx, resp.ID, container.StartOptions{})) |
| 83 | + |
| 84 | + serverStart := make(chan serverStartResult) |
| 85 | + go func() { |
| 86 | + prOut, pwOut := io.Pipe() |
| 87 | + prErr, pwErr := io.Pipe() |
| 88 | + |
| 89 | + go func() { |
| 90 | + // Ignore error, we should be done? |
| 91 | + // TODO: maybe check for use of closed network connection specifically |
| 92 | + _, _ = stdcopy.StdCopy(pwOut, pwErr, hijackedResponse.Reader) |
| 93 | + pwOut.Close() |
| 94 | + pwErr.Close() |
| 95 | + }() |
| 96 | + |
| 97 | + bufferedStderr := bufio.NewReader(prErr) |
| 98 | + line, err := bufferedStderr.ReadString('\n') |
| 99 | + if err != nil { |
| 100 | + serverStart <- serverStartResult{err: err} |
| 101 | + } |
| 102 | + |
| 103 | + if strings.TrimSpace(line) != "GitHub MCP Server running on stdio" { |
| 104 | + serverStart <- serverStartResult{ |
| 105 | + err: fmt.Errorf("unexpected server output: %s", line), |
| 106 | + } |
| 107 | + return |
| 108 | + } |
| 109 | + |
| 110 | + serverStart <- serverStartResult{ |
| 111 | + server: server{ |
| 112 | + m: m, |
| 113 | + log: testLogWriter{t}, |
| 114 | + stdin: hijackedResponse.Conn, |
| 115 | + stdout: bufio.NewReader(prOut), |
| 116 | + }, |
| 117 | + } |
| 118 | + }() |
| 119 | + |
| 120 | + t.Logf("waiting for %s server to start...", m) |
| 121 | + serveResult := <-serverStart |
| 122 | + require.NoError(t, serveResult.err, "expected the server to start successfully") |
| 123 | + |
| 124 | + return serveResult.server |
| 125 | +} |
| 126 | + |
| 127 | +func TestCapabilities(t *testing.T) { |
| 128 | + anthropicServer := start(t, anthropic) |
| 129 | + githubServer := start(t, github) |
| 130 | + |
| 131 | + req := newInitializeRequest( |
| 132 | + initializeRequestParams{ |
| 133 | + ProtocolVersion: "2024-11-05", |
| 134 | + Capabilities: clientCapabilities{}, |
| 135 | + ClientInfo: clientInfo{ |
| 136 | + Name: "ConformanceTest", |
| 137 | + Version: "0.0.1", |
| 138 | + }, |
| 139 | + }, |
| 140 | + ) |
| 141 | + |
| 142 | + require.NoError(t, anthropicServer.send(req)) |
| 143 | + |
| 144 | + var anthropicInitializeResponse initializeResponse |
| 145 | + require.NoError(t, anthropicServer.receive(&anthropicInitializeResponse)) |
| 146 | + |
| 147 | + require.NoError(t, githubServer.send(req)) |
| 148 | + |
| 149 | + var ghInitializeResponse initializeResponse |
| 150 | + require.NoError(t, githubServer.receive(&ghInitializeResponse)) |
| 151 | + |
| 152 | + // Any capabilities in the anthropic response should be present in the github response |
| 153 | + // (though the github response may have additional capabilities) |
| 154 | + if diff := diffNonNilFields(anthropicInitializeResponse.Result.Capabilities, ghInitializeResponse.Result.Capabilities, ""); diff != "" { |
| 155 | + t.Errorf("capabilities mismatch:\n%s", diff) |
| 156 | + } |
| 157 | +} |
| 158 | + |
| 159 | +func diffNonNilFields(a, b interface{}, path string) string { |
| 160 | + var sb strings.Builder |
| 161 | + |
| 162 | + va := reflect.ValueOf(a) |
| 163 | + vb := reflect.ValueOf(b) |
| 164 | + |
| 165 | + if !va.IsValid() { |
| 166 | + return "" |
| 167 | + } |
| 168 | + |
| 169 | + if va.Kind() == reflect.Ptr { |
| 170 | + if va.IsNil() { |
| 171 | + return "" |
| 172 | + } |
| 173 | + if !vb.IsValid() || vb.IsNil() { |
| 174 | + sb.WriteString(path + "\n") |
| 175 | + return sb.String() |
| 176 | + } |
| 177 | + va = va.Elem() |
| 178 | + vb = vb.Elem() |
| 179 | + } |
| 180 | + |
| 181 | + if va.Kind() != reflect.Struct || vb.Kind() != reflect.Struct { |
| 182 | + return "" |
| 183 | + } |
| 184 | + |
| 185 | + t := va.Type() |
| 186 | + for i := range va.NumField() { |
| 187 | + field := t.Field(i) |
| 188 | + if !field.IsExported() { |
| 189 | + continue |
| 190 | + } |
| 191 | + |
| 192 | + subPath := field.Name |
| 193 | + if path != "" { |
| 194 | + subPath = fmt.Sprintf("%s.%s", path, field.Name) |
| 195 | + } |
| 196 | + |
| 197 | + fieldA := va.Field(i) |
| 198 | + fieldB := vb.Field(i) |
| 199 | + |
| 200 | + switch fieldA.Kind() { |
| 201 | + case reflect.Ptr: |
| 202 | + if fieldA.IsNil() { |
| 203 | + continue // not required |
| 204 | + } |
| 205 | + if fieldB.IsNil() { |
| 206 | + sb.WriteString(subPath + "\n") |
| 207 | + continue |
| 208 | + } |
| 209 | + sb.WriteString(diffNonNilFields(fieldA.Interface(), fieldB.Interface(), subPath)) |
| 210 | + |
| 211 | + case reflect.Struct: |
| 212 | + sb.WriteString(diffNonNilFields(fieldA.Interface(), fieldB.Interface(), subPath)) |
| 213 | + |
| 214 | + default: |
| 215 | + zero := reflect.Zero(fieldA.Type()) |
| 216 | + if !reflect.DeepEqual(fieldA.Interface(), zero.Interface()) { |
| 217 | + // fieldA is non-zero; now check that fieldB matches |
| 218 | + if !reflect.DeepEqual(fieldA.Interface(), fieldB.Interface()) { |
| 219 | + sb.WriteString(subPath + "\n") |
| 220 | + } |
| 221 | + } |
| 222 | + } |
| 223 | + } |
| 224 | + |
| 225 | + return sb.String() |
| 226 | +} |
| 227 | + |
| 228 | +type serverStartResult struct { |
| 229 | + server server |
| 230 | + err error |
| 231 | +} |
| 232 | + |
| 233 | +type server struct { |
| 234 | + m maintainer |
| 235 | + log io.Writer |
| 236 | + |
| 237 | + stdin io.Writer |
| 238 | + stdout *bufio.Reader |
| 239 | +} |
| 240 | + |
| 241 | +func (s server) send(req request) error { |
| 242 | + b, err := req.marshal() |
| 243 | + if err != nil { |
| 244 | + return err |
| 245 | + } |
| 246 | + |
| 247 | + fmt.Fprintf(s.log, "sending %s: %s\n", s.m, string(b)) |
| 248 | + |
| 249 | + n, err := s.stdin.Write(append(b, '\n')) |
| 250 | + if err != nil { |
| 251 | + return err |
| 252 | + } |
| 253 | + |
| 254 | + if n != len(b)+1 { |
| 255 | + return fmt.Errorf("wrote %d bytes, expected %d", n, len(b)+1) |
| 256 | + } |
| 257 | + |
| 258 | + return nil |
| 259 | +} |
| 260 | + |
| 261 | +func (s server) receive(res response) error { |
| 262 | + line, err := s.stdout.ReadBytes('\n') |
| 263 | + if err != nil { |
| 264 | + if err == io.EOF { |
| 265 | + return fmt.Errorf("EOF after reading %s", string(line)) |
| 266 | + } |
| 267 | + return err |
| 268 | + } |
| 269 | + |
| 270 | + fmt.Fprintf(s.log, "received from %s: %s\n", s.m, string(line)) |
| 271 | + |
| 272 | + return res.unmarshal(line) |
| 273 | +} |
| 274 | + |
| 275 | +type jsonRPRCRequest[params any] struct { |
| 276 | + JSONRPC string `json:"jsonrpc"` |
| 277 | + ID int `json:"id"` |
| 278 | + Method string `json:"method"` |
| 279 | + Params params `json:"params"` |
| 280 | +} |
| 281 | + |
| 282 | +type jsonRPRCResponse[result any] struct { |
| 283 | + JSONRPC string `json:"jsonrpc"` |
| 284 | + ID int `json:"id"` |
| 285 | + Method string `json:"method"` |
| 286 | + Result result `json:"result"` |
| 287 | +} |
| 288 | + |
| 289 | +type request interface { |
| 290 | + marshal() ([]byte, error) |
| 291 | +} |
| 292 | + |
| 293 | +type response interface { |
| 294 | + unmarshal([]byte) error |
| 295 | +} |
| 296 | + |
| 297 | +func newInitializeRequest(params initializeRequestParams) initializeRequest { |
| 298 | + return initializeRequest{ |
| 299 | + jsonRPRCRequest: jsonRPRCRequest[initializeRequestParams]{ |
| 300 | + JSONRPC: "2.0", |
| 301 | + ID: 1, |
| 302 | + Method: "initialize", |
| 303 | + Params: params, |
| 304 | + }, |
| 305 | + } |
| 306 | +} |
| 307 | + |
| 308 | +type initializeRequest struct { |
| 309 | + jsonRPRCRequest[initializeRequestParams] |
| 310 | +} |
| 311 | + |
| 312 | +func (r initializeRequest) marshal() ([]byte, error) { |
| 313 | + return json.Marshal(r) |
| 314 | +} |
| 315 | + |
| 316 | +type initializeRequestParams struct { |
| 317 | + ProtocolVersion string `json:"protocolVersion"` |
| 318 | + Capabilities clientCapabilities `json:"capabilities"` |
| 319 | + ClientInfo clientInfo `json:"clientInfo"` |
| 320 | +} |
| 321 | + |
| 322 | +type clientCapabilities struct{} // don't actually care about any of these right now |
| 323 | + |
| 324 | +type clientInfo struct { |
| 325 | + Name string `json:"name"` |
| 326 | + Version string `json:"version"` |
| 327 | +} |
| 328 | + |
| 329 | +type initializeResponse struct { |
| 330 | + jsonRPRCResponse[initializeResult] |
| 331 | +} |
| 332 | + |
| 333 | +func (r *initializeResponse) unmarshal(b []byte) error { |
| 334 | + return json.Unmarshal(b, r) |
| 335 | +} |
| 336 | + |
| 337 | +type initializeResult struct { |
| 338 | + ProtocolVersion string `json:"protocolVersion"` |
| 339 | + Capabilities serverCapabilities `json:"capabilities"` |
| 340 | + ServerInfo serverInfo `json:"serverInfo"` |
| 341 | +} |
| 342 | + |
| 343 | +type serverCapabilities struct { |
| 344 | + Logging *struct{} `json:"logging,omitempty"` |
| 345 | + Prompts *struct { |
| 346 | + ListChanged bool `json:"listChanged,omitempty"` |
| 347 | + } `json:"prompts,omitempty"` |
| 348 | + Resources *struct { |
| 349 | + Subscribe bool `json:"subscribe,omitempty"` |
| 350 | + ListChanged bool `json:"listChanged,omitempty"` |
| 351 | + } `json:"resources,omitempty"` |
| 352 | + Tools *struct { |
| 353 | + ListChanged bool `json:"listChanged,omitempty"` |
| 354 | + } `json:"tools,omitempty"` |
| 355 | +} |
| 356 | + |
| 357 | +type serverInfo struct { |
| 358 | + Name string `json:"name"` |
| 359 | + Version string `json:"version"` |
| 360 | +} |
0 commit comments